underpost 3.2.80 → 3.2.90
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +182 -1
- package/CLI-HELP.md +37 -16
- package/README.md +2 -2
- package/bin/deploy.js +18 -16
- package/docker-compose.yml +1 -1
- package/manifests/cronjobs/dd-cron/dd-cron-backup.yaml +1 -1
- package/manifests/cronjobs/dd-cron/dd-cron-dns.yaml +1 -1
- package/manifests/deployment/dd-default-development/deployment.yaml +2 -2
- package/manifests/deployment/playwright/deployment.yaml +1 -1
- package/manifests/mongodb/kustomization.yaml +4 -1
- package/manifests/mongodb/statefulset.yaml +4 -0
- package/manifests/mongodb/storage-class.yaml +9 -2
- package/package.json +17 -17
- package/scripts/nat-iptables.sh +10 -4
- package/scripts/test-monitor.sh +4 -3
- package/src/cli/cluster.js +740 -55
- package/src/cli/db.js +2 -2
- package/src/cli/deploy.js +1679 -174
- package/src/cli/docker-compose.js +19 -178
- package/src/cli/image.js +15 -6
- package/src/cli/index.js +124 -35
- package/src/cli/ipfs.js +82 -11
- package/src/cli/monitor.js +1 -1
- package/src/cli/repository.js +1 -1
- package/src/cli/run.js +2161 -420
- package/src/cli/secrets.js +969 -0
- package/src/cli/ssh.js +8 -28
- package/src/client-builder/client-build.js +94 -11
- package/src/client-builder/ssr.js +27 -73
- package/src/db/mongo/MongoBootstrap.js +295 -54
- package/src/db/mongo/MongooseDB.js +47 -32
- package/src/index.js +1 -1
- package/src/server/conf.js +1208 -70
- package/src/server/cri.js +70 -0
- package/src/server/underpost-gateway.js +1073 -0
- package/src/server/underpost-ingress.js +364 -0
- package/test/cluster-instances.test.js +435 -0
- package/test/deploy-node-placement.test.js +45 -0
- package/test/instance-traffic-plan.test.js +710 -0
- package/test/sops-secret-store.test.js +612 -0
- package/test/underpost-gateway.test.js +469 -0
- package/test/underpost-ingress.test.js +253 -0
package/src/cli/run.js
CHANGED
|
@@ -5,78 +5,74 @@
|
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
7
|
import { daemonProcess, getTerminalPid, pbcopy, shellCd, shellExec } from '../server/process.js';
|
|
8
|
-
import crypto from 'crypto';
|
|
9
8
|
import {
|
|
10
9
|
awaitDeployMonitor,
|
|
11
10
|
buildKindPorts,
|
|
12
|
-
|
|
11
|
+
clusterContextFactory,
|
|
12
|
+
clusterTypeFactory,
|
|
13
13
|
cronDeployIdResolve,
|
|
14
|
+
dispatchBuildInstanceEnv,
|
|
15
|
+
deployHostsFactory,
|
|
14
16
|
etcHostFactory,
|
|
17
|
+
exposePartialMatchesFactory,
|
|
18
|
+
exposePathPartsFactory,
|
|
19
|
+
exposePortListFactory,
|
|
20
|
+
exposePortPlanFactory,
|
|
21
|
+
exposeTcpPortsFactory,
|
|
22
|
+
gatewayApiEnabledFactory,
|
|
23
|
+
generateSecurePassword,
|
|
15
24
|
getNpmRootPath,
|
|
25
|
+
instanceHttpRouteRulesFactory,
|
|
26
|
+
instanceInterceptStatusesFactory,
|
|
27
|
+
instancePortFactory,
|
|
28
|
+
instanceProxyRoutesFactory,
|
|
29
|
+
instanceStatusPageEntriesFactory,
|
|
16
30
|
isDeployRunnerContext,
|
|
17
31
|
loadConfInstances,
|
|
32
|
+
loadProjectInstanceEnvBuilder,
|
|
18
33
|
loadConfServerJson,
|
|
19
|
-
selectConfInstances,
|
|
20
34
|
loadReplicas,
|
|
35
|
+
resolveDeployList,
|
|
36
|
+
resolveEnvScoped,
|
|
37
|
+
selectConfInstances,
|
|
38
|
+
waitForPort,
|
|
21
39
|
writeEnv,
|
|
40
|
+
clusterInstancesFactory,
|
|
41
|
+
deployTrafficEntriesFactory,
|
|
42
|
+
curlStatusChainFactory,
|
|
43
|
+
hostIngressFactsFactory,
|
|
44
|
+
hostRenderInstancesFactory,
|
|
45
|
+
instanceTrafficPlanFactory,
|
|
46
|
+
trafficTableRowsFactory,
|
|
47
|
+
isTrafficServingFactory,
|
|
48
|
+
nextTrafficFactory,
|
|
49
|
+
stopPlanFactory,
|
|
50
|
+
trafficFromRoutingInfoFactory,
|
|
22
51
|
} from '../server/conf.js';
|
|
23
52
|
import { actionInitLog, loggerFactory } from '../server/logger.js';
|
|
24
53
|
|
|
25
54
|
import fs from 'fs-extra';
|
|
26
|
-
import net from 'net';
|
|
27
55
|
import { range, s4, setPad, timer } from '../client/components/core/CommonJs.js';
|
|
28
56
|
|
|
29
57
|
import os from 'os';
|
|
30
58
|
import Underpost from '../index.js';
|
|
31
59
|
import dotenv from 'dotenv';
|
|
32
60
|
import { MongoBootstrap } from '../db/mongo/MongoBootstrap.js';
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
setTimeout(tryConnect, interval);
|
|
47
|
-
});
|
|
48
|
-
};
|
|
49
|
-
tryConnect();
|
|
50
|
-
});
|
|
51
|
-
|
|
61
|
+
import {
|
|
62
|
+
UNDERPOST_GATEWAY,
|
|
63
|
+
assertStaticAssets,
|
|
64
|
+
gatewayFallbackProbeRunner,
|
|
65
|
+
hostServerConfFactory,
|
|
66
|
+
installGatewayConf,
|
|
67
|
+
instanceFallbackChecksFactory,
|
|
68
|
+
placeInstanceStaticAssets,
|
|
69
|
+
pwaFallbackChecksFactory,
|
|
70
|
+
readHostInstanceRegistry,
|
|
71
|
+
writeHostInstanceRegistry,
|
|
72
|
+
writeHostServerConf,
|
|
73
|
+
} from '../server/underpost-gateway.js';
|
|
52
74
|
const logger = loggerFactory(import.meta);
|
|
53
75
|
|
|
54
|
-
/**
|
|
55
|
-
* @method instanceProxyRoutesFactory
|
|
56
|
-
* @description Renders the HTTPProxy route block for every instance sharing a host.
|
|
57
|
-
* Routes are emitted longest-prefix first so a specific instance path (`/FOREST`)
|
|
58
|
-
* is never shadowed by the default instance's catch-all (`/`).
|
|
59
|
-
* @param {string} deployId - Parent deployment identifier.
|
|
60
|
-
* @param {Array<object>} instances - Expanded instance entries bound to one host.
|
|
61
|
-
* @param {string} env - `development` | `production`.
|
|
62
|
-
* @param {Object<string,string>} trafficById - Instance id → traffic colour.
|
|
63
|
-
* @returns {string} Concatenated route YAML.
|
|
64
|
-
*/
|
|
65
|
-
const instanceProxyRoutesFactory = ({ deployId, instances, env, trafficById }) =>
|
|
66
|
-
[...instances]
|
|
67
|
-
.sort((a, b) => (b.path || '/').length - (a.path || '/').length)
|
|
68
|
-
.map((instance) =>
|
|
69
|
-
Underpost.deploy.deploymentYamlServiceFactory({
|
|
70
|
-
path: instance.path,
|
|
71
|
-
port: env === 'development' && instance.fromDebugPort ? instance.fromDebugPort : instance.fromPort,
|
|
72
|
-
deployId: `${deployId}-${instance.id}`,
|
|
73
|
-
env,
|
|
74
|
-
deploymentVersions: [trafficById[instance.id] || 'blue'],
|
|
75
|
-
pathRewritePolicy: instance.pathRewritePolicy,
|
|
76
|
-
}),
|
|
77
|
-
)
|
|
78
|
-
.join('');
|
|
79
|
-
|
|
80
76
|
/**
|
|
81
77
|
* @constant DEFAULT_OPTION
|
|
82
78
|
* @description Default options for the UnderpostRun class.
|
|
@@ -85,8 +81,12 @@ const instanceProxyRoutesFactory = ({ deployId, instances, env, trafficById }) =
|
|
|
85
81
|
* @property {boolean} dev - Whether to run in development mode.
|
|
86
82
|
* @property {string} podName - The name of the pod to run.
|
|
87
83
|
* @property {string} nodeName - The name of the node to run.
|
|
84
|
+
* @property {string} ingressNode - Dedicated node for the host-network public ingress; never inherited from nodeName.
|
|
88
85
|
* @property {string} sshKeyPath - Private key path for node SSH operations, forwarded to volume shipping over SSH.
|
|
89
86
|
* @property {number} port - Custom port to use.
|
|
87
|
+
* @property {string} exposeContainerPorts - Comma-separated Service/container destination ports.
|
|
88
|
+
* @property {string} exposeHostPorts - Comma-separated host listening ports.
|
|
89
|
+
* @property {boolean} localProxy - Start the development path proxy after exposing matched resources.
|
|
90
90
|
* @property {string} volumeHostPath - The host path for the volume.
|
|
91
91
|
* @property {string} volumeMountPath - The mount path for the volume.
|
|
92
92
|
* @property {string} imageName - The name of the image to run.
|
|
@@ -101,6 +101,11 @@ const instanceProxyRoutesFactory = ({ deployId, instances, env, trafficById }) =
|
|
|
101
101
|
* @property {boolean} force - Whether to force the operation.
|
|
102
102
|
* @property {boolean} reset - Whether to reset the operation.
|
|
103
103
|
* @property {boolean} tls - Whether to use TLS.
|
|
104
|
+
* @property {boolean} gatewayApi - Apply the Gateway API stack (Gateway + HTTPRoute) instead of the Contour HTTPProxy. Both manifest sets are always generated.
|
|
105
|
+
* @property {boolean} disableGatewayApi - Fall back to the Contour HTTPProxy stack in runners where the Gateway API is the default (`cluster`).
|
|
106
|
+
* @property {string} gatewayClass - GatewayClass name baked into generated Gateway manifests.
|
|
107
|
+
* @property {boolean} disableHttp3 - Omit QUIC/HTTP3 listener config and the Alt-Svc advertisement.
|
|
108
|
+
* @property {number} quicPort - UDP port advertised for QUIC/HTTP3.
|
|
104
109
|
* @property {string} cmd - The command to run in the container.
|
|
105
110
|
* @property {string} tty - The TTY option for the container.
|
|
106
111
|
* @property {string} stdin - The stdin option for the container.
|
|
@@ -151,6 +156,18 @@ const instanceProxyRoutesFactory = ({ deployId, instances, env, trafficById }) =
|
|
|
151
156
|
* @property {boolean} pullBundle - Whether to pull the bundle before running. Use together with --skip-full-build to skip the local build entirely (supported by: sync, template-deploy).
|
|
152
157
|
* @property {boolean} remove - Whether to remove/teardown resources instead of creating them (e.g. delete-expose for k3s proxy devices in dev-cluster).
|
|
153
158
|
* @property {boolean} test - Whether to enable test/generic-purpose mode (e.g. use self-signed TLS instead of cert-manager).
|
|
159
|
+
* @property {string} hostAliases - Pod `/etc/hosts` entries, as semicolon-separated `ip=host1,host2` groups.
|
|
160
|
+
* @property {string} args - Comma-separated arguments forwarded to the runner's own command.
|
|
161
|
+
* @property {boolean} cert - Issue cert-manager certificates; set implicitly by `tls` in the promote workflow.
|
|
162
|
+
* @property {boolean} instanceOnly - Act on the one instance id given, without expanding its variant family.
|
|
163
|
+
* @property {string} labels - Comma-separated `key=value` pairs applied to the created resources.
|
|
164
|
+
* @property {string} npmRoot - Resolved npm global root, cached on the options once looked up.
|
|
165
|
+
* @property {object} on - Lifecycle hooks (`{ init }`) a programmatic caller supplies; unused from the CLI.
|
|
166
|
+
* @property {string} traffic - Blue/green colour to bake into generated manifests (default: `blue`).
|
|
167
|
+
* @property {boolean} gatewayBootstrapComplete - Internal marker: a parent orchestration already proved the static gateway fallback.
|
|
168
|
+
* @property {boolean} noBackendCheckpoint - Internal marker: this promote is the deliberate no-backend fallback checkpoint, so it must not wait for the target colour's endpoints.
|
|
169
|
+
* @property {Object<string,string>} targetTrafficById - Internal instance id → explicitly pre-routed traffic colour.
|
|
170
|
+
* @property {string} volumeType - hostPath volume type (`DirectoryOrCreate`, `FileOrCreate`, or `dev` for the latter).
|
|
154
171
|
* @property {string} branch - The Git branch to use for operations (e.g., for template-deploy, ssh-deploy).
|
|
155
172
|
* @memberof UnderpostRun
|
|
156
173
|
*/
|
|
@@ -158,8 +175,12 @@ const DEFAULT_OPTION = {
|
|
|
158
175
|
dev: false,
|
|
159
176
|
podName: '',
|
|
160
177
|
nodeName: '',
|
|
178
|
+
ingressNode: '',
|
|
161
179
|
sshKeyPath: '',
|
|
162
180
|
port: 0,
|
|
181
|
+
exposeContainerPorts: '',
|
|
182
|
+
exposeHostPorts: '',
|
|
183
|
+
localProxy: false,
|
|
163
184
|
volumeHostPath: '',
|
|
164
185
|
volumeMountPath: '',
|
|
165
186
|
imageName: '',
|
|
@@ -174,6 +195,11 @@ const DEFAULT_OPTION = {
|
|
|
174
195
|
force: false,
|
|
175
196
|
reset: false,
|
|
176
197
|
tls: false,
|
|
198
|
+
gatewayApi: false,
|
|
199
|
+
disableGatewayApi: false,
|
|
200
|
+
gatewayClass: '',
|
|
201
|
+
disableHttp3: false,
|
|
202
|
+
quicPort: 0,
|
|
177
203
|
cmd: '',
|
|
178
204
|
tty: '',
|
|
179
205
|
stdin: '',
|
|
@@ -223,6 +249,17 @@ const DEFAULT_OPTION = {
|
|
|
223
249
|
remove: false,
|
|
224
250
|
test: false,
|
|
225
251
|
branch: '',
|
|
252
|
+
args: '',
|
|
253
|
+
cert: false,
|
|
254
|
+
instanceOnly: false,
|
|
255
|
+
gatewayBootstrapComplete: false,
|
|
256
|
+
noBackendCheckpoint: false,
|
|
257
|
+
targetTrafficById: {},
|
|
258
|
+
labels: '',
|
|
259
|
+
npmRoot: '',
|
|
260
|
+
on: undefined,
|
|
261
|
+
traffic: '',
|
|
262
|
+
volumeType: '',
|
|
226
263
|
};
|
|
227
264
|
|
|
228
265
|
/**
|
|
@@ -234,6 +271,34 @@ const DEFAULT_OPTION = {
|
|
|
234
271
|
* runners for executing specific commands.
|
|
235
272
|
* @memberof UnderpostRun
|
|
236
273
|
*/
|
|
274
|
+
|
|
275
|
+
// Secrets `sops-setup` onboards when no explicit list is passed: the full self-hosted data tier.
|
|
276
|
+
// `mongodb-keyfile` is listed alongside `mongodb-secret` because the MongoDB StatefulSet mounts
|
|
277
|
+
// it as a volume for intra-replica-set auth and will not start without it, so onboarding the
|
|
278
|
+
// credentials alone would leave Mongo broken.
|
|
279
|
+
const SOPS_SETUP_DEFAULT_SECRETS = ['postgres-secret', 'mariadb-secret', 'mongodb-secret', 'mongodb-keyfile'];
|
|
280
|
+
|
|
281
|
+
/**
|
|
282
|
+
* Produces a value for a Secret data key that has no origin seed file and no `--args` override.
|
|
283
|
+
* Key-aware because the data tier does not want one shape of secret: a replica-set keyfile is a
|
|
284
|
+
* long base64 blob, a username is an identifier, and everything else is a password.
|
|
285
|
+
* @param {string} key - Secret data key (e.g. 'password', 'username', 'mongodb-keyfile').
|
|
286
|
+
* @returns {string} Generated value.
|
|
287
|
+
* @memberof UnderpostRun
|
|
288
|
+
*/
|
|
289
|
+
const generateSeedValue = (key) => {
|
|
290
|
+
if (key === 'username') return 'admin';
|
|
291
|
+
// MongoDB keyfile: 6-1024 base64 characters shared by every replica-set member. Newlines are
|
|
292
|
+
// stripped so the value round-trips identically through YAML and through
|
|
293
|
+
// MongoBootstrap.readCredential, which strips them too.
|
|
294
|
+
if (key === 'mongodb-keyfile')
|
|
295
|
+
return shellExec(`openssl rand -base64 756`, { stdout: true, silent: true, disableLog: true }).replace(
|
|
296
|
+
/\r?\n/g,
|
|
297
|
+
'',
|
|
298
|
+
);
|
|
299
|
+
return generateSecurePassword(24);
|
|
300
|
+
};
|
|
301
|
+
|
|
237
302
|
class UnderpostRun {
|
|
238
303
|
/**
|
|
239
304
|
* @static
|
|
@@ -242,6 +307,180 @@ class UnderpostRun {
|
|
|
242
307
|
* @memberof UnderpostRun
|
|
243
308
|
*/
|
|
244
309
|
static RUNNERS = {
|
|
310
|
+
/**
|
|
311
|
+
* @method status
|
|
312
|
+
* @description Reports deployment traffic, routing, Pods, expanded instances, and host capacity.
|
|
313
|
+
* @param {string} path - Deploy id, comma-separated ids, or `dd`; empty uses the router/configured projects.
|
|
314
|
+
* @param {UnderpostRunDefaultOptions} options - Namespace, environment (`--dev`), cluster, and node options.
|
|
315
|
+
* @returns {Promise<{deployments: object[], machine: object}>} Structured status report.
|
|
316
|
+
* @memberof UnderpostRun
|
|
317
|
+
*/
|
|
318
|
+
status: async (path = '', options = DEFAULT_OPTION) => {
|
|
319
|
+
options = {
|
|
320
|
+
...options,
|
|
321
|
+
gatewayApi: gatewayApiEnabledFactory(options),
|
|
322
|
+
namespace: options.namespace || 'default',
|
|
323
|
+
};
|
|
324
|
+
if (!/^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$/.test(options.namespace))
|
|
325
|
+
throw new Error(`Invalid Kubernetes namespace: ${options.namespace}`);
|
|
326
|
+
if (options.nodeName && !/^[a-zA-Z0-9._-]+$/.test(options.nodeName))
|
|
327
|
+
throw new Error(`Invalid Kubernetes node name: ${options.nodeName}`);
|
|
328
|
+
const env = options.dev ? 'development' : 'production';
|
|
329
|
+
const requestedDeploys = `${path || options.deployId || ''}`.trim();
|
|
330
|
+
const routerPath = './engine-private/deploy/dd.router';
|
|
331
|
+
const confRoot = './engine-private/conf';
|
|
332
|
+
const deployIds = [
|
|
333
|
+
...new Set(
|
|
334
|
+
requestedDeploys
|
|
335
|
+
? resolveDeployList(requestedDeploys)
|
|
336
|
+
: fs.existsSync(routerPath)
|
|
337
|
+
? resolveDeployList('dd')
|
|
338
|
+
: fs.existsSync(confRoot)
|
|
339
|
+
? fs
|
|
340
|
+
.readdirSync(confRoot)
|
|
341
|
+
.filter(
|
|
342
|
+
(deployId) =>
|
|
343
|
+
fs.existsSync(`${confRoot}/${deployId}/conf.server.json`) ||
|
|
344
|
+
fs.existsSync(`${confRoot}/${deployId}/conf.instances.json`),
|
|
345
|
+
)
|
|
346
|
+
.sort()
|
|
347
|
+
: [],
|
|
348
|
+
),
|
|
349
|
+
];
|
|
350
|
+
if (deployIds.length === 0) throw new Error('No deployments found for status');
|
|
351
|
+
if (deployIds.some((deployId) => !/^[a-zA-Z0-9._-]+$/.test(deployId)))
|
|
352
|
+
throw new Error(`Invalid deployment status path: ${requestedDeploys}`);
|
|
353
|
+
|
|
354
|
+
const deployments = [];
|
|
355
|
+
for (const deployId of deployIds) {
|
|
356
|
+
const instances = [];
|
|
357
|
+
if (fs.existsSync(`${confRoot}/${deployId}/conf.instances.json`)) {
|
|
358
|
+
for (const instance of loadConfInstances(deployId)) {
|
|
359
|
+
const instanceDeployId = `${deployId}-${instance.id}`;
|
|
360
|
+
instances.push({
|
|
361
|
+
id: instance.id,
|
|
362
|
+
host: instance.host,
|
|
363
|
+
path: instance.path,
|
|
364
|
+
fromPort: instance.fromPort,
|
|
365
|
+
toPort: instance.toPort,
|
|
366
|
+
fromDebugPort: instance.fromDebugPort,
|
|
367
|
+
toDebugPort: instance.toDebugPort,
|
|
368
|
+
traffic: Underpost.deploy.getCurrentTraffic(instanceDeployId, {
|
|
369
|
+
namespace: options.namespace,
|
|
370
|
+
hostTest: instance.host,
|
|
371
|
+
env,
|
|
372
|
+
gatewayApi: options.gatewayApi,
|
|
373
|
+
}),
|
|
374
|
+
});
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
const deployment = {
|
|
378
|
+
deployId,
|
|
379
|
+
env,
|
|
380
|
+
traffic: Underpost.deploy.getCurrentTraffic(deployId, {
|
|
381
|
+
namespace: options.namespace,
|
|
382
|
+
env,
|
|
383
|
+
gatewayApi: options.gatewayApi,
|
|
384
|
+
}),
|
|
385
|
+
router: await Underpost.deploy.routerFactory(deployId, env),
|
|
386
|
+
pods: Underpost.kubectl.get(deployId, 'pods', options.namespace),
|
|
387
|
+
instances,
|
|
388
|
+
};
|
|
389
|
+
deployments.push(deployment);
|
|
390
|
+
logger.info('', deployment);
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
const interfaceName = Underpost.dns.getDefaultNetworkInterface();
|
|
394
|
+
const machine = {
|
|
395
|
+
hostname: os.hostname(),
|
|
396
|
+
arch: Underpost.baremetal.getHostArch(),
|
|
397
|
+
clusterType: clusterTypeFactory(options),
|
|
398
|
+
ipv4Public: await Underpost.dns.getPublicIp(),
|
|
399
|
+
ipv4Local: Underpost.dns.getLocalIPv4Address(),
|
|
400
|
+
resources: Underpost.cluster.getResourcesCapacity(options.nodeName),
|
|
401
|
+
defaultInterfaceName: interfaceName,
|
|
402
|
+
defaultInterfaceInfo: os.networkInterfaces()[interfaceName],
|
|
403
|
+
};
|
|
404
|
+
logger.info('Machine', machine);
|
|
405
|
+
return { deployments, machine };
|
|
406
|
+
},
|
|
407
|
+
|
|
408
|
+
/**
|
|
409
|
+
* @method expose
|
|
410
|
+
* @description Port-forwards every Service whose name partially matches path, falling back to matching Pods.
|
|
411
|
+
* Works through the active kubeconfig for Kind, k3s, and kubeadm clusters.
|
|
412
|
+
* Comma-separated path fragments determine resource index order; host and
|
|
413
|
+
* container port lists are paired against that same order.
|
|
414
|
+
* @param {string} path - One or more comma-separated literal Service/Pod name fragments.
|
|
415
|
+
* @param {UnderpostRunDefaultOptions} options - Namespace, cluster type, and optional port overrides.
|
|
416
|
+
* @returns {Array<{kindType: string, name: string, localPort: number, remotePort: number}>} Forward plan.
|
|
417
|
+
* @memberof UnderpostRun
|
|
418
|
+
*/
|
|
419
|
+
expose: (path, options = DEFAULT_OPTION) => {
|
|
420
|
+
const namespace = options.namespace || 'default';
|
|
421
|
+
const clusterType = clusterTypeFactory(options);
|
|
422
|
+
const pathParts = exposePathPartsFactory(path || options.podName);
|
|
423
|
+
if (!/^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$/.test(namespace))
|
|
424
|
+
throw new Error(`Invalid Kubernetes namespace: ${namespace}`);
|
|
425
|
+
let kindType = 'svc';
|
|
426
|
+
let resources = exposePartialMatchesFactory(Underpost.kubectl.get('', kindType, namespace), pathParts);
|
|
427
|
+
|
|
428
|
+
if (resources.length === 0) {
|
|
429
|
+
kindType = 'pod';
|
|
430
|
+
resources = exposePartialMatchesFactory(Underpost.kubectl.get('', 'pods', namespace), pathParts);
|
|
431
|
+
}
|
|
432
|
+
if (resources.length === 0)
|
|
433
|
+
throw new Error(`No Service or Pod partially matching '${pathParts.join(',')}' in namespace '${namespace}'`);
|
|
434
|
+
|
|
435
|
+
const containerPorts = exposePortListFactory(options.exposeContainerPorts, '--expose-container-ports');
|
|
436
|
+
const hostPorts = exposePortListFactory(options.exposeHostPorts, '--expose-host-ports');
|
|
437
|
+
const portsOf = (resource) => {
|
|
438
|
+
let declaredPorts = exposeTcpPortsFactory(resource);
|
|
439
|
+
if (kindType === 'pod' && declaredPorts.length === 0) {
|
|
440
|
+
const podJson = shellExec(`sudo kubectl get pod ${resource.NAME} -n ${namespace} -o json`, {
|
|
441
|
+
stdout: true,
|
|
442
|
+
silent: true,
|
|
443
|
+
});
|
|
444
|
+
const pod = JSON.parse(podJson);
|
|
445
|
+
declaredPorts = (pod.spec?.containers || [])
|
|
446
|
+
.flatMap((container) => container.ports || [])
|
|
447
|
+
.map(({ containerPort }) => parseInt(containerPort))
|
|
448
|
+
.filter((port) => Number.isInteger(port) && port > 0);
|
|
449
|
+
}
|
|
450
|
+
return declaredPorts;
|
|
451
|
+
};
|
|
452
|
+
const plan = exposePortPlanFactory({ resources, kindType, containerPorts, hostPorts, portsOf });
|
|
453
|
+
|
|
454
|
+
logger.info('[expose] Kubernetes port-forward plan', {
|
|
455
|
+
clusterType,
|
|
456
|
+
namespace,
|
|
457
|
+
matches: pathParts,
|
|
458
|
+
plan,
|
|
459
|
+
});
|
|
460
|
+
for (const { kindType, name, localPort, remotePort } of plan)
|
|
461
|
+
shellExec(`sudo kubectl port-forward -n ${namespace} ${kindType}/${name} ${localPort}:${remotePort}`, {
|
|
462
|
+
async: true,
|
|
463
|
+
});
|
|
464
|
+
|
|
465
|
+
if (options.localProxy) {
|
|
466
|
+
const deployId = options.deployId || pathParts[0];
|
|
467
|
+
const env = options.dev ? 'development' : 'production';
|
|
468
|
+
const envFile = `./engine-private/conf/${deployId}/.env.${env}`;
|
|
469
|
+
let basePort = plan[0].localPort - 1;
|
|
470
|
+
if (fs.existsSync(envFile)) {
|
|
471
|
+
const portMatch = fs.readFileSync(envFile, 'utf8').match(/^PORT=(\d+)/m);
|
|
472
|
+
if (portMatch) basePort = parseInt(portMatch[1]);
|
|
473
|
+
}
|
|
474
|
+
const tlsFlag = options.tls ? ' tls' : '';
|
|
475
|
+
shellExec(
|
|
476
|
+
`NODE_ENV=${env} PORT=${basePort} DEV_PROXY_PORT_OFFSET=0 node src/proxy proxy ${deployId} ${env}${tlsFlag}`,
|
|
477
|
+
{ async: true },
|
|
478
|
+
);
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
return plan;
|
|
482
|
+
},
|
|
483
|
+
|
|
245
484
|
/**
|
|
246
485
|
* @method dev-cluster
|
|
247
486
|
* @description Resets and deploys a full development cluster including MongoDB, Valkey, exposes services, and updates `/etc/hosts` for local access.
|
|
@@ -253,26 +492,21 @@ class UnderpostRun {
|
|
|
253
492
|
const baseCommand = options.dev ? 'node bin' : 'underpost';
|
|
254
493
|
const mongoHosts = ['mongodb-0.mongodb-service'];
|
|
255
494
|
let primaryMongoHost = 'mongodb-0.mongodb-service';
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
495
|
+
const clusterType = clusterTypeFactory(options);
|
|
496
|
+
const clusterFlag = ` --${clusterType}`;
|
|
497
|
+
const clusterInitFlag = clusterType === 'kind' ? '' : clusterFlag;
|
|
498
|
+
const clusterOptions = `${options.dev ? ' --dev' : ''}${clusterInitFlag} --namespace ${options.namespace}`;
|
|
499
|
+
if (!options.expose && !options.remove) {
|
|
500
|
+
shellExec(`${baseCommand} cluster${clusterOptions} --reset`);
|
|
501
|
+
shellExec(`${baseCommand} cluster${clusterOptions}`);
|
|
259
502
|
|
|
260
503
|
shellExec(
|
|
261
|
-
`${baseCommand} cluster${
|
|
262
|
-
',',
|
|
263
|
-
)} --pull-image`,
|
|
504
|
+
`${baseCommand} cluster${clusterOptions} --mongodb --service-host ${mongoHosts.join(',')} --pull-image`,
|
|
264
505
|
);
|
|
265
|
-
shellExec(`${baseCommand} cluster${
|
|
506
|
+
shellExec(`${baseCommand} cluster${clusterOptions} --valkey --pull-image`);
|
|
266
507
|
}
|
|
267
|
-
if (options.
|
|
268
|
-
|
|
269
|
-
shellExec(`${baseCommand} lxd --delete-expose k3s-control:27017`);
|
|
270
|
-
shellExec(`${baseCommand} lxd --delete-expose k3s-control:6379`);
|
|
271
|
-
} else {
|
|
272
|
-
shellExec(`${baseCommand} lxd --expose k3s-control:27017 --node-port 32017`);
|
|
273
|
-
shellExec(`${baseCommand} lxd --expose k3s-control:6379 --node-port 32079`);
|
|
274
|
-
}
|
|
275
|
-
shellExec(`lxc config device show k3s-control`);
|
|
508
|
+
if (options.remove) {
|
|
509
|
+
shellExec(`${baseCommand} run kill '6379,27017'`);
|
|
276
510
|
} else {
|
|
277
511
|
try {
|
|
278
512
|
const primaryPodName =
|
|
@@ -281,49 +515,26 @@ class UnderpostRun {
|
|
|
281
515
|
podName: 'mongodb-0',
|
|
282
516
|
disableAuth: options.dev,
|
|
283
517
|
}) || 'mongodb-0';
|
|
284
|
-
|
|
285
|
-
`${baseCommand} deploy --expose --namespace ${options.namespace} --disable-update-underpost-config mongo`,
|
|
286
|
-
{ async: true },
|
|
287
|
-
);
|
|
288
|
-
shellExec(
|
|
289
|
-
`${baseCommand} deploy --expose --namespace ${options.namespace} --disable-update-underpost-config valkey`,
|
|
290
|
-
{ async: true },
|
|
291
|
-
);
|
|
518
|
+
primaryMongoHost = `${primaryPodName}.mongodb-service`;
|
|
292
519
|
} catch (error) {
|
|
293
520
|
logger.warn('Failed to detect MongoDB primary pod, using default', {
|
|
294
521
|
error: error.message,
|
|
295
522
|
default: primaryMongoHost,
|
|
296
523
|
});
|
|
297
524
|
}
|
|
525
|
+
shellExec(
|
|
526
|
+
`${baseCommand} run expose mongodb-service --namespace ${options.namespace}${clusterFlag} --expose-container-ports 27017 --expose-host-ports 27017`,
|
|
527
|
+
{ async: true },
|
|
528
|
+
);
|
|
529
|
+
shellExec(
|
|
530
|
+
`${baseCommand} run expose valkey-service --namespace ${options.namespace}${clusterFlag} --expose-container-ports 6379 --expose-host-ports 6379`,
|
|
531
|
+
{ async: true },
|
|
532
|
+
);
|
|
298
533
|
}
|
|
299
534
|
const hostListenResult = etcHostFactory([primaryMongoHost]);
|
|
300
535
|
logger.info(hostListenResult.renderHosts);
|
|
301
536
|
},
|
|
302
537
|
|
|
303
|
-
/**
|
|
304
|
-
* @method etc-hosts
|
|
305
|
-
* @description Modifies the `/etc/hosts` file to add entries for local access to services,
|
|
306
|
-
* based on the provided path input.
|
|
307
|
-
* @param {string} path - The input value, identifier, or path for the operation (used to specify the entries to add to /etc/hosts).
|
|
308
|
-
*/
|
|
309
|
-
'etc-hosts': (path = '', options = DEFAULT_OPTION) => {
|
|
310
|
-
etcHostFactory(path.split(','));
|
|
311
|
-
},
|
|
312
|
-
|
|
313
|
-
/**
|
|
314
|
-
* @method ipfs-expose
|
|
315
|
-
* @description Exposes IPFS Cluster services on specified ports for local access.
|
|
316
|
-
* @type {Function}
|
|
317
|
-
* @memberof UnderpostRun
|
|
318
|
-
*/
|
|
319
|
-
'ipfs-expose': (path, options = DEFAULT_OPTION) => {
|
|
320
|
-
const ports = [5001, 9094, 8080];
|
|
321
|
-
for (const port of ports)
|
|
322
|
-
shellExec(`node bin deploy --expose ipfs-cluster --expose-port ${port} --disable-update-underpost-config`, {
|
|
323
|
-
async: true,
|
|
324
|
-
});
|
|
325
|
-
},
|
|
326
|
-
|
|
327
538
|
/**
|
|
328
539
|
* @method metadata
|
|
329
540
|
* @description Generates metadata for the specified path after exposing the development cluster.
|
|
@@ -336,19 +547,30 @@ class UnderpostRun {
|
|
|
336
547
|
shellExec(`node bin run kill '${ports}'`);
|
|
337
548
|
shellExec(`node bin run dev-cluster --dev --expose --namespace ${options.namespace}`, { async: true });
|
|
338
549
|
logger.info('Waiting for port-forward services to be ready...');
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
logger.info('Port-forward services are ready');
|
|
342
|
-
} catch (err) {
|
|
343
|
-
logger.error('Port-forward services failed to become ready', { error: err.message });
|
|
550
|
+
const ready = await Promise.all([27017, 6379].map((port) => waitForPort({ port })));
|
|
551
|
+
if (ready.some((reachable) => !reachable)) {
|
|
344
552
|
shellExec(`node bin run kill '${ports}'`);
|
|
345
|
-
throw
|
|
553
|
+
throw new Error('Port-forward services failed to become ready');
|
|
346
554
|
}
|
|
555
|
+
logger.info('Port-forward services are ready');
|
|
347
556
|
shellExec(`node bin metadata --generate ${path}`);
|
|
348
557
|
shellExec(`node bin db --dev --clean-fs-collection dd`);
|
|
349
558
|
shellExec(`node bin run kill '${ports}'`);
|
|
350
559
|
},
|
|
351
560
|
|
|
561
|
+
/**
|
|
562
|
+
* @method ipfs-expose
|
|
563
|
+
* @description Exposes every declared TCP port on the matching IPFS Cluster Service.
|
|
564
|
+
* @type {Function}
|
|
565
|
+
* @memberof UnderpostRun
|
|
566
|
+
*/
|
|
567
|
+
'ipfs-expose': (path, options = DEFAULT_OPTION) => {
|
|
568
|
+
// 5001 Kubo RPC API / WebUI: Kubo/IPFS HTTP API + IPFS WebUI (e.g., http://localhost:5001/webui).
|
|
569
|
+
// 9094 IPFS Cluster HTTP API: IPFS Cluster REST API consumed by ipfs-cluster-ctl and WebUI clients (e.g., http://localhost:9094/).
|
|
570
|
+
// 8080 (or 8081) IPFS Gateway: Public HTTP Gateway for accessing pinned content by CID (e.g., http://localhost:8080/ipfs/Qm...).
|
|
571
|
+
shellExec(`node bin run expose ipfs --expose-host-ports 5001,9094,8080`);
|
|
572
|
+
},
|
|
573
|
+
|
|
352
574
|
/**
|
|
353
575
|
* @method svc-ls
|
|
354
576
|
* @description Lists systemd services and installed packages, optionally filtering by the provided path.
|
|
@@ -384,28 +606,6 @@ class UnderpostRun {
|
|
|
384
606
|
shellExec(`sudo rm -f /etc/yum.repos.d/${path}*.repo`);
|
|
385
607
|
},
|
|
386
608
|
|
|
387
|
-
/**
|
|
388
|
-
* @method ssh-deploy-info
|
|
389
|
-
* @description Retrieves deployment status and pod information from a remote server via SSH.
|
|
390
|
-
* @param {string} path - The input value, identifier, or path for the operation.
|
|
391
|
-
* @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
|
|
392
|
-
* @memberof UnderpostRun
|
|
393
|
-
*/
|
|
394
|
-
'ssh-deploy-info': async (path = '', options = DEFAULT_OPTION) => {
|
|
395
|
-
const env = options.dev ? 'development' : 'production';
|
|
396
|
-
await Underpost.ssh.sshRemoteRunner(
|
|
397
|
-
`node bin deploy ${path ? path : 'dd'} ${env} --status && kubectl get pods -A`,
|
|
398
|
-
{
|
|
399
|
-
deployId: options.deployId,
|
|
400
|
-
user: options.user,
|
|
401
|
-
dev: options.dev,
|
|
402
|
-
remote: true,
|
|
403
|
-
useSudo: true,
|
|
404
|
-
cd: '/home/dd/engine',
|
|
405
|
-
},
|
|
406
|
-
);
|
|
407
|
-
},
|
|
408
|
-
|
|
409
609
|
/**
|
|
410
610
|
* @method node-move
|
|
411
611
|
* @description Abstract runner that relocates any schedulable Kubernetes workload
|
|
@@ -475,8 +675,8 @@ class UnderpostRun {
|
|
|
475
675
|
services: 'service',
|
|
476
676
|
})[k] || k;
|
|
477
677
|
|
|
478
|
-
// Kinds that own a pod template we can patch
|
|
479
|
-
//
|
|
678
|
+
// Kinds that own a pod template we can patch. Changing that template is
|
|
679
|
+
// itself the controller's rollout trigger.
|
|
480
680
|
const templated = [
|
|
481
681
|
'deployment',
|
|
482
682
|
'statefulset',
|
|
@@ -486,7 +686,6 @@ class UnderpostRun {
|
|
|
486
686
|
'cronjob',
|
|
487
687
|
'replicationcontroller',
|
|
488
688
|
];
|
|
489
|
-
const rolloutKinds = ['deployment', 'statefulset', 'daemonset'];
|
|
490
689
|
const templateSelectorPath = (kind) =>
|
|
491
690
|
kind === 'cronjob'
|
|
492
691
|
? ['spec', 'jobTemplate', 'spec', 'template', 'spec', 'nodeSelector']
|
|
@@ -575,10 +774,9 @@ class UnderpostRun {
|
|
|
575
774
|
continue;
|
|
576
775
|
}
|
|
577
776
|
|
|
578
|
-
// Idempotency: skip the patch
|
|
579
|
-
//
|
|
580
|
-
//
|
|
581
|
-
// rollout restart.
|
|
777
|
+
// Idempotency: skip the patch if the resource is already where we want
|
|
778
|
+
// it. Compares the live pod-template nodeSelector against the desired
|
|
779
|
+
// placement so a repeated run does not trigger an unnecessary rollout.
|
|
582
780
|
const basePath = kind === 'cronjob' ? 'spec.jobTemplate.spec.template.spec' : 'spec.template.spec';
|
|
583
781
|
const jsonpath = (expr) =>
|
|
584
782
|
shellExec(`kubectl get ${kind} ${name} -n ${ns} -o jsonpath='${expr}'`, {
|
|
@@ -608,16 +806,17 @@ class UnderpostRun {
|
|
|
608
806
|
}
|
|
609
807
|
|
|
610
808
|
const patchCmd = `kubectl patch ${kind} ${name} -n ${ns} --type=merge -p '${buildPatch(kind)}'`;
|
|
611
|
-
const restartCmd = `kubectl rollout restart ${kind} ${name} -n ${ns}`;
|
|
612
809
|
if (dryRun) {
|
|
613
810
|
logger.info(`[dry-run] ${patchCmd}`);
|
|
614
|
-
if (rolloutKinds.includes(kind)) logger.info(`[dry-run] ${restartCmd}`);
|
|
615
811
|
results.push({ ref, kind, status: 'dry-run', node: remove ? undefined : node });
|
|
616
812
|
continue;
|
|
617
813
|
}
|
|
618
814
|
|
|
619
815
|
shellExec(patchCmd);
|
|
620
|
-
|
|
816
|
+
// nodeSelector is part of the pod template, so this patch already
|
|
817
|
+
// creates a new controller revision. A rollout restart here creates a
|
|
818
|
+
// second, immediately superseding revision and can strand the previous
|
|
819
|
+
// Ready replica in "pending termination" while the newest pod starts.
|
|
621
820
|
logger.info(remove ? `Cleared node placement: ${kind}/${name}` : `Moved ${kind}/${name} -> ${node}`, {
|
|
622
821
|
namespace: ns,
|
|
623
822
|
});
|
|
@@ -635,10 +834,12 @@ class UnderpostRun {
|
|
|
635
834
|
* @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
|
|
636
835
|
* @memberof UnderpostRun
|
|
637
836
|
*/
|
|
638
|
-
'dev-hosts-expose': (path, options = DEFAULT_OPTION) => {
|
|
639
|
-
shellExec(
|
|
640
|
-
|
|
641
|
-
|
|
837
|
+
'dev-hosts-expose': async (path, options = DEFAULT_OPTION) => {
|
|
838
|
+
shellExec(`node bin deploy ${path} development --disable-update-deployment --disable-update-proxy --kubeadm`);
|
|
839
|
+
// /etc/hosts is written here, not by `deploy`: that command has no
|
|
840
|
+
// --etc-hosts option, and the `etc-hosts` runner already resolves a
|
|
841
|
+
// deploy's hosts from its conf.server.json.
|
|
842
|
+
await UnderpostRun.RUNNERS['etc-hosts']('', { ...options, deployId: path });
|
|
642
843
|
},
|
|
643
844
|
|
|
644
845
|
/**
|
|
@@ -649,7 +850,10 @@ class UnderpostRun {
|
|
|
649
850
|
* @memberof UnderpostRun
|
|
650
851
|
*/
|
|
651
852
|
'dev-hosts-restore': (path, options = DEFAULT_OPTION) => {
|
|
652
|
-
|
|
853
|
+
// Rewrite /etc/hosts with the loopback block alone, dropping the deploy
|
|
854
|
+
// host entries `dev-hosts-expose` (and the `cluster` runner) added.
|
|
855
|
+
const hostListenResult = etcHostFactory([]);
|
|
856
|
+
logger.info(hostListenResult.renderHosts);
|
|
653
857
|
},
|
|
654
858
|
|
|
655
859
|
/**
|
|
@@ -935,6 +1139,7 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
|
|
|
935
1139
|
sync: async (path, options = DEFAULT_OPTION) => {
|
|
936
1140
|
// Dev usage: node bin run --dev --build sync dd-default
|
|
937
1141
|
const env = options.dev ? 'development' : 'production';
|
|
1142
|
+
options = { ...options, gatewayApi: gatewayApiEnabledFactory(options) };
|
|
938
1143
|
const baseCommand = 'node bin'; // options.dev ? 'node bin' : 'underpost';
|
|
939
1144
|
const baseClusterCommand = options.dev ? ' --dev' : '';
|
|
940
1145
|
const clusterFlag = options.k3s ? ' --k3s' : options.kind ? ' --kind' : ' --kubeadm';
|
|
@@ -972,11 +1177,28 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
|
|
|
972
1177
|
}
|
|
973
1178
|
|
|
974
1179
|
const currentTraffic = isDeployRunnerContext(path, options)
|
|
975
|
-
? Underpost.deploy.getCurrentTraffic(deployId, {
|
|
1180
|
+
? Underpost.deploy.getCurrentTraffic(deployId, {
|
|
1181
|
+
namespace: options.namespace,
|
|
1182
|
+
env,
|
|
1183
|
+
gatewayApi: options.gatewayApi,
|
|
1184
|
+
})
|
|
976
1185
|
: '';
|
|
977
1186
|
let targetTraffic = currentTraffic ? (currentTraffic === 'blue' ? 'green' : 'blue') : 'green';
|
|
978
1187
|
if (targetTraffic) versions = versions ? versions : targetTraffic;
|
|
979
1188
|
|
|
1189
|
+
// The routed colour is only live traffic while it still has a ready
|
|
1190
|
+
// endpoint. Everything downstream that would take the host offline to prove
|
|
1191
|
+
// the maintenance fallback is conditional on this being false: a first
|
|
1192
|
+
// bring-up has nothing to interrupt, a re-deploy does.
|
|
1193
|
+
const serving = isTrafficServingFactory({
|
|
1194
|
+
liveTraffic: currentTraffic,
|
|
1195
|
+
hasReadyEndpoints: (colour) =>
|
|
1196
|
+
Underpost.deploy.serviceHasReadyEndpoints({
|
|
1197
|
+
service: `${deployId}-${env}-${colour}-service`,
|
|
1198
|
+
namespace: options.namespace || 'default',
|
|
1199
|
+
}),
|
|
1200
|
+
});
|
|
1201
|
+
|
|
980
1202
|
const ignorePods =
|
|
981
1203
|
isDeployRunnerContext(path, options) && targetTraffic
|
|
982
1204
|
? Underpost.kubectl.get(`${deployId}-${env}-${targetTraffic}`, 'pods', options.namespace).map((p) => p.NAME)
|
|
@@ -992,16 +1214,69 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
|
|
|
992
1214
|
const pullBundleFlag = options.pullBundle ? ' --pull-bundle' : '';
|
|
993
1215
|
const imagePullPolicyFlag = options.imagePullPolicy ? ` --image-pull-policy ${options.imagePullPolicy}` : '';
|
|
994
1216
|
const sshKeyPathFlag = options.sshKeyPath ? ` --ssh-key-path ${options.sshKeyPath}` : '';
|
|
1217
|
+
const gatewayApiFlags = Underpost.deploy.gatewayApiFlagsFactory(options);
|
|
1218
|
+
|
|
1219
|
+
// A direct sync owns the same gateway-first contract as the full cluster
|
|
1220
|
+
// runner. Build the host-side SSR documents before generating routes unless
|
|
1221
|
+
// the caller explicitly selected a pre-built bundle workflow.
|
|
1222
|
+
if (isDeployRunnerContext(path, options) && !options.skipFullBuild)
|
|
1223
|
+
shellExec(`${baseCommand} client ${deployId} ${env}`);
|
|
995
1224
|
|
|
996
1225
|
shellExec(
|
|
997
1226
|
`${baseCommand} deploy${clusterFlag} --build-manifest --sync --info-router --replicas ${replicas} --node ${node}${
|
|
998
1227
|
image ? ` --image ${image}` : ''
|
|
999
1228
|
}${versions ? ` --versions ${versions}` : ''}${
|
|
1000
1229
|
options.namespace ? ` --namespace ${options.namespace}` : ''
|
|
1001
|
-
}${timeoutFlags}${cmdString}${gitCleanFlag}${skipFullBuildFlag}${pullBundleFlag}${imagePullPolicyFlag}${sshKeyPathFlag} ${deployId} ${env}`,
|
|
1230
|
+
}${timeoutFlags}${cmdString}${gitCleanFlag}${skipFullBuildFlag}${pullBundleFlag}${imagePullPolicyFlag}${sshKeyPathFlag}${gatewayApiFlags} ${deployId} ${env}`,
|
|
1002
1231
|
);
|
|
1003
1232
|
|
|
1004
1233
|
if (isDeployRunnerContext(path, options)) {
|
|
1234
|
+
if (options.gatewayApi) {
|
|
1235
|
+
const namespace = options.namespace || 'default';
|
|
1236
|
+
const gatewayRoot = Underpost.deploy.underpostGatewayRootFactory(options);
|
|
1237
|
+
shellExec(`kubectl rollout status deployment/${UNDERPOST_GATEWAY.name} -n ${namespace} --timeout=5m`);
|
|
1238
|
+
const staticAssets = Underpost.deploy.syncStaticAssets(deployId, env, {
|
|
1239
|
+
...options,
|
|
1240
|
+
namespace,
|
|
1241
|
+
// Prefer the currently serving workload as the document source. On
|
|
1242
|
+
// an initial deploy there is none, so sync falls back to the checkout.
|
|
1243
|
+
versions: currentTraffic || versions || targetTraffic,
|
|
1244
|
+
});
|
|
1245
|
+
assertStaticAssets({ records: staticAssets, hostRoot: gatewayRoot, label: 'sync' });
|
|
1246
|
+
|
|
1247
|
+
if (serving)
|
|
1248
|
+
logger.info('[sync] Live colour serving; holding traffic until the target colour is Ready', {
|
|
1249
|
+
deployId,
|
|
1250
|
+
live: currentTraffic,
|
|
1251
|
+
target: targetTraffic,
|
|
1252
|
+
});
|
|
1253
|
+
else {
|
|
1254
|
+
// Remove any inactive-colour workload left by an earlier cycle. Without
|
|
1255
|
+
// this cleanup the probe could hit a stale Ready pod and never exercise
|
|
1256
|
+
// the configured unavailable-backend fallback.
|
|
1257
|
+
shellExec(
|
|
1258
|
+
`kubectl delete service ${deployId}-${env}-${targetTraffic}-service -n ${namespace} --ignore-not-found`,
|
|
1259
|
+
);
|
|
1260
|
+
shellExec(
|
|
1261
|
+
`kubectl delete deployment ${deployId}-${env}-${targetTraffic} -n ${namespace} --ignore-not-found`,
|
|
1262
|
+
);
|
|
1263
|
+
// Publish the target-colour route while its Service is deliberately
|
|
1264
|
+
// absent. Site paths reach underpost-gateway and must return the
|
|
1265
|
+
// configured maintenance body before deployment.yaml is submitted.
|
|
1266
|
+
shellExec(
|
|
1267
|
+
`${baseCommand} deploy${clusterFlag}${cmdString} --replicas ${replicas} --node ${node} --disable-update-deployment ${deployId} ${env} --versions ${versions}${
|
|
1268
|
+
options.namespace ? ` --namespace ${options.namespace}` : ''
|
|
1269
|
+
}${timeoutFlags}${gitCleanFlag}${imagePullPolicyFlag}${sshKeyPathFlag}${gatewayApiFlags}`,
|
|
1270
|
+
);
|
|
1271
|
+
await gatewayFallbackProbeRunner({
|
|
1272
|
+
gatewayStatusRunner: UnderpostRun.RUNNERS['gateway-status'],
|
|
1273
|
+
checks: pwaFallbackChecksFactory(deployId),
|
|
1274
|
+
options,
|
|
1275
|
+
label: 'sync',
|
|
1276
|
+
});
|
|
1277
|
+
}
|
|
1278
|
+
}
|
|
1279
|
+
|
|
1005
1280
|
// Backup app/services repositories with repo-backup configured
|
|
1006
1281
|
shellExec(
|
|
1007
1282
|
`${baseCommand} db ${deployId} ${clusterFlag}${baseClusterCommand} --repo-backup --primary-pod --git --force-clone --preserveUUID ${options.namespace ? ` --ns ${options.namespace}` : ''}`,
|
|
@@ -1009,163 +1284,119 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
|
|
|
1009
1284
|
shellExec(
|
|
1010
1285
|
`${baseCommand} deploy${clusterFlag}${cmdString} --replicas ${replicas} --node ${node} --disable-update-proxy ${deployId} ${env} --versions ${versions}${
|
|
1011
1286
|
options.namespace ? ` --namespace ${options.namespace}` : ''
|
|
1012
|
-
}${timeoutFlags}${gitCleanFlag}${imagePullPolicyFlag}${sshKeyPathFlag}`,
|
|
1287
|
+
}${timeoutFlags}${gitCleanFlag}${imagePullPolicyFlag}${sshKeyPathFlag}${gatewayApiFlags}`,
|
|
1013
1288
|
);
|
|
1014
1289
|
if (!targetTraffic)
|
|
1015
|
-
targetTraffic = Underpost.deploy.getCurrentTraffic(deployId, {
|
|
1290
|
+
targetTraffic = Underpost.deploy.getCurrentTraffic(deployId, {
|
|
1291
|
+
namespace: options.namespace,
|
|
1292
|
+
env,
|
|
1293
|
+
gatewayApi: options.gatewayApi,
|
|
1294
|
+
});
|
|
1016
1295
|
await Underpost.monitor.monitorReadyRunner(deployId, env, targetTraffic, ignorePods, options.namespace);
|
|
1017
1296
|
Underpost.deploy.switchTraffic(deployId, env, targetTraffic, replicas, options.namespace, options);
|
|
1018
1297
|
} else
|
|
1019
|
-
logger.info(
|
|
1298
|
+
logger.info(
|
|
1299
|
+
'current traffic',
|
|
1300
|
+
Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace, env }),
|
|
1301
|
+
);
|
|
1020
1302
|
},
|
|
1021
1303
|
|
|
1022
1304
|
/**
|
|
1023
1305
|
* @method stop
|
|
1024
|
-
* @description
|
|
1025
|
-
*
|
|
1306
|
+
* @description Deletes colour-suffixed Deployments and their Services, leaving routing untouched.
|
|
1307
|
+
*
|
|
1308
|
+
* Four ways to say what to stop, resolved by {@link ServerConfBuilder.stopPlanFactory}:
|
|
1309
|
+
*
|
|
1310
|
+
* ```bash
|
|
1311
|
+
* # Literal: exactly this Deployment, flags ignored
|
|
1312
|
+
* node bin run stop dd-cyberia-mmo-server-forest-development-blue
|
|
1313
|
+
*
|
|
1314
|
+
* # The deploy's PWA workload, inactive colour
|
|
1315
|
+
* node bin run stop --deploy-id dd-cyberia
|
|
1316
|
+
*
|
|
1317
|
+
* # ...plus every variant of each instance family, inactive colour
|
|
1318
|
+
* node bin run stop --deploy-id dd-cyberia --instance-id mmo-client,mmo-server
|
|
1319
|
+
*
|
|
1320
|
+
* # Explicit colours; both, where they exist
|
|
1321
|
+
* node bin run stop --deploy-id dd-cyberia --traffic blue,green
|
|
1322
|
+
* ```
|
|
1323
|
+
*
|
|
1324
|
+
* The default colour is the blue/green partner of whatever each target is
|
|
1325
|
+
* serving, so a stop is safe against a live host unless `--traffic` names the
|
|
1326
|
+
* serving colour outright — which is warned about, not refused.
|
|
1327
|
+
*
|
|
1328
|
+
* That colour is read through the same routing stack that published it: the
|
|
1329
|
+
* Gateway API HTTPRoute by default, the Contour HTTPProxy under
|
|
1330
|
+
* `--disable-gateway-api`. Reading the wrong kind finds no colour, degrades to
|
|
1331
|
+
* "blue", and stops whichever Deployment happens to carry that name. It is
|
|
1332
|
+
* resolved per target too — an instance's colour lives under its own
|
|
1333
|
+
* `<deployId>-<instanceId>` prefix, and on a shared host the parent's answer
|
|
1334
|
+
* belongs to a different variant.
|
|
1335
|
+
* @param {string} [path] - Literal comma-separated Deployment names; when set, every flag is ignored.
|
|
1026
1336
|
* @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
|
|
1027
1337
|
* @memberof UnderpostRun
|
|
1028
1338
|
*/
|
|
1029
1339
|
stop: async (path = '', options = DEFAULT_OPTION) => {
|
|
1030
|
-
let currentTraffic = Underpost.deploy.getCurrentTraffic(options.deployId, {
|
|
1031
|
-
namespace: options.namespace,
|
|
1032
|
-
hostTest: options.hosts,
|
|
1033
|
-
});
|
|
1034
1340
|
const env = options.dev ? 'development' : 'production';
|
|
1341
|
+
const namespace = options.namespace || 'default';
|
|
1342
|
+
const gatewayApi = gatewayApiEnabledFactory(options);
|
|
1343
|
+
|
|
1344
|
+
// Memoized because the plan and the serving check ask the same question,
|
|
1345
|
+
// and `--traffic` skips the plan's lookup entirely — which is exactly when
|
|
1346
|
+
// the check has to make its own.
|
|
1347
|
+
const liveTraffic = {};
|
|
1348
|
+
const liveTrafficOf = (target) => {
|
|
1349
|
+
if (liveTraffic[target.id] === undefined)
|
|
1350
|
+
liveTraffic[target.id] =
|
|
1351
|
+
Underpost.deploy.getCurrentTraffic(target.id, {
|
|
1352
|
+
hostTest: target.host || options.hosts,
|
|
1353
|
+
namespace,
|
|
1354
|
+
env,
|
|
1355
|
+
gatewayApi,
|
|
1356
|
+
}) || '';
|
|
1357
|
+
return liveTraffic[target.id];
|
|
1358
|
+
};
|
|
1035
1359
|
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
const deploymentId = `${_deployId ? _deployId : options.deployId}${
|
|
1039
|
-
options.instanceId ? `-${options.instanceId}` : ''
|
|
1040
|
-
}-${env}-${currentTraffic}`;
|
|
1041
|
-
|
|
1042
|
-
shellExec(`kubectl delete deployment ${deploymentId} -n ${options.namespace}`);
|
|
1043
|
-
shellExec(`kubectl delete svc ${deploymentId}-service -n ${options.namespace}`);
|
|
1044
|
-
},
|
|
1045
|
-
|
|
1046
|
-
/**
|
|
1047
|
-
* @method ssh-deploy-stop
|
|
1048
|
-
* @description Stops a remote deployment via SSH by executing the appropriate Underpost command on the remote server.
|
|
1049
|
-
* @param {string} path - The input value, identifier, or path for the operation (used to determine which traffic to stop).
|
|
1050
|
-
* @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
|
|
1051
|
-
* @memberof UnderpostRun
|
|
1052
|
-
*/
|
|
1053
|
-
'ssh-deploy-stop': async (path, options = DEFAULT_OPTION) => {
|
|
1054
|
-
const baseCommand = options.dev ? 'node bin' : 'underpost';
|
|
1055
|
-
const baseClusterCommand = options.dev ? ' --dev' : '';
|
|
1056
|
-
|
|
1057
|
-
const remoteCommand = [
|
|
1058
|
-
`${baseCommand} run${baseClusterCommand} stop${path ? ` ${path}` : ''}`,
|
|
1059
|
-
` --deploy-id ${options.deployId}${options.instanceId ? ` --instance-id ${options.instanceId}` : ''}`,
|
|
1060
|
-
` --namespace ${options.namespace}${options.hosts ? ` --hosts ${options.hosts}` : ''}`,
|
|
1061
|
-
].join('');
|
|
1062
|
-
|
|
1063
|
-
await Underpost.ssh.sshRemoteRunner(remoteCommand, {
|
|
1064
|
-
deployId: options.deployId,
|
|
1065
|
-
user: options.user,
|
|
1066
|
-
dev: options.dev,
|
|
1067
|
-
remote: true,
|
|
1068
|
-
useSudo: true,
|
|
1069
|
-
cd: '/home/dd/engine',
|
|
1070
|
-
});
|
|
1071
|
-
},
|
|
1072
|
-
|
|
1073
|
-
/**
|
|
1074
|
-
* @method ssh-deploy-db-rollback
|
|
1075
|
-
* @description Performs a database rollback on remote deployment via SSH.
|
|
1076
|
-
* @param {string} path - Comma-separated deployId and optional number of commits to reset (format: "deployId,nCommits")
|
|
1077
|
-
* @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
|
|
1078
|
-
* @param {string} options.deployId - The deployment identifier
|
|
1079
|
-
* @param {string} options.user - The SSH user for credential lookup
|
|
1080
|
-
* @param {boolean} options.dev - Development mode flag
|
|
1081
|
-
* @memberof UnderpostRun
|
|
1082
|
-
*/
|
|
1083
|
-
'ssh-deploy-db-rollback': async (path = '', options = DEFAULT_OPTION) => {
|
|
1084
|
-
const baseCommand = options.dev ? 'node bin' : 'underpost';
|
|
1085
|
-
let [deployId, nCommitsReset] = path.split(',');
|
|
1086
|
-
if (!nCommitsReset) nCommitsReset = 1;
|
|
1087
|
-
|
|
1088
|
-
const remoteCommand = `${baseCommand} db ${deployId} --git --kubeadm --primary-pod --force-clone --macro-rollback-export ${nCommitsReset}${options.namespace ? ` --ns ${options.namespace}` : ''}`;
|
|
1089
|
-
|
|
1090
|
-
await Underpost.ssh.sshRemoteRunner(remoteCommand, {
|
|
1091
|
-
deployId: options.deployId,
|
|
1092
|
-
user: options.user,
|
|
1093
|
-
dev: options.dev,
|
|
1094
|
-
remote: true,
|
|
1095
|
-
useSudo: true,
|
|
1096
|
-
cd: '/home/dd/engine',
|
|
1097
|
-
});
|
|
1098
|
-
},
|
|
1099
|
-
|
|
1100
|
-
/**
|
|
1101
|
-
* @method ssh-deploy-db
|
|
1102
|
-
* @description Imports/restores a database on remote deployment via SSH.
|
|
1103
|
-
* @param {string} path - The deployment ID for database import
|
|
1104
|
-
* @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
|
|
1105
|
-
* @param {string} options.deployId - The deployment identifier
|
|
1106
|
-
* @param {string} options.user - The SSH user for credential lookup
|
|
1107
|
-
* @param {boolean} options.dev - Development mode flag
|
|
1108
|
-
* @memberof UnderpostRun
|
|
1109
|
-
*/
|
|
1110
|
-
'ssh-deploy-db': async (path, options = DEFAULT_OPTION) => {
|
|
1111
|
-
const baseCommand = options.dev ? 'node bin' : 'underpost';
|
|
1112
|
-
|
|
1113
|
-
const remoteCommand = `${baseCommand} db ${path} --import --drop --preserveUUID --git --kubeadm --primary-pod --force-clone${options.namespace ? ` --ns ${options.namespace}` : ''}`;
|
|
1114
|
-
|
|
1115
|
-
await Underpost.ssh.sshRemoteRunner(remoteCommand, {
|
|
1360
|
+
const { deployments, error } = stopPlanFactory({
|
|
1361
|
+
path,
|
|
1116
1362
|
deployId: options.deployId,
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1363
|
+
instanceId: options.instanceId,
|
|
1364
|
+
traffic: options.traffic,
|
|
1365
|
+
env,
|
|
1366
|
+
instancesFor: (instanceId) => selectConfInstances(loadConfInstances(options.deployId), instanceId),
|
|
1367
|
+
liveTrafficOf,
|
|
1122
1368
|
});
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
* @method ssh-deploy-db-status
|
|
1127
|
-
* @description Retrieves database status/stats for a deployment (or all deployments from dd.router) via SSH.
|
|
1128
|
-
* @param {string} path - Comma-separated deployId(s) or 'dd' to use the dd.router list.
|
|
1129
|
-
* @param {UnderpostRunDefaultOptions} options - Runner options (uses options.deployId for SSH host lookup).
|
|
1130
|
-
* @param {string} options.deployId - Deployment identifier used for SSH config lookup.
|
|
1131
|
-
* @param {string} options.user - SSH user for credential lookup.
|
|
1132
|
-
* @param {boolean} options.dev - Development mode flag.
|
|
1133
|
-
* @param {string} [options.namespace] - Kubernetes namespace to pass to the db check.
|
|
1134
|
-
* @memberof UnderpostRun
|
|
1135
|
-
*/
|
|
1136
|
-
'ssh-deploy-db-status': async (path = '', options = DEFAULT_OPTION) => {
|
|
1137
|
-
const baseCommand = options.dev ? 'node bin' : 'underpost';
|
|
1138
|
-
|
|
1139
|
-
let deployList = [];
|
|
1140
|
-
if (!path || path === 'dd') {
|
|
1141
|
-
if (!fs.existsSync('./engine-private/deploy/dd.router')) {
|
|
1142
|
-
logger.warn('dd.router not found; nothing to run');
|
|
1143
|
-
return;
|
|
1144
|
-
}
|
|
1145
|
-
deployList = fs
|
|
1146
|
-
.readFileSync('./engine-private/deploy/dd.router', 'utf8')
|
|
1147
|
-
.split(',')
|
|
1148
|
-
.map((d) => d.trim())
|
|
1149
|
-
.filter(Boolean);
|
|
1150
|
-
} else {
|
|
1151
|
-
deployList = path
|
|
1152
|
-
.split(',')
|
|
1153
|
-
.map((d) => d.trim())
|
|
1154
|
-
.filter(Boolean);
|
|
1369
|
+
if (error) {
|
|
1370
|
+
logger.error(error);
|
|
1371
|
+
return [];
|
|
1155
1372
|
}
|
|
1156
1373
|
|
|
1157
|
-
|
|
1158
|
-
|
|
1374
|
+
// Read once: it decides what is reported as actually stopped, and a target
|
|
1375
|
+
// that is already gone is a no-op worth naming rather than a silent delete.
|
|
1376
|
+
const deployedNames = Underpost.kubectl.get('', 'deployment', namespace).map((entry) => entry.NAME);
|
|
1377
|
+
const stopped = [];
|
|
1378
|
+
for (const target of deployments) {
|
|
1379
|
+
const existed = deployedNames.includes(target.deployment);
|
|
1380
|
+
if (existed) stopped.push(target.deployment);
|
|
1381
|
+
// The Service is deleted either way: an orphan left by a half-finished
|
|
1382
|
+
// cycle outlives its Deployment and would keep resolving to no endpoints.
|
|
1383
|
+
shellExec(`kubectl delete deployment ${target.deployment} -n ${namespace} --ignore-not-found`);
|
|
1384
|
+
shellExec(`kubectl delete svc ${target.deployment}-service -n ${namespace} --ignore-not-found`);
|
|
1385
|
+
}
|
|
1159
1386
|
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
cd: '/home/dd/engine',
|
|
1387
|
+
const serving = deployments.filter(
|
|
1388
|
+
(target) => target.kind !== 'literal' && target.colour === liveTrafficOf(target),
|
|
1389
|
+
);
|
|
1390
|
+
if (serving.length > 0)
|
|
1391
|
+
logger.warn('Stopped the serving colour; those routes stay published and now have no backend', {
|
|
1392
|
+
deployments: serving.map((target) => target.deployment),
|
|
1167
1393
|
});
|
|
1168
|
-
|
|
1394
|
+
logger.info('Stop complete', {
|
|
1395
|
+
namespace,
|
|
1396
|
+
stopped,
|
|
1397
|
+
absent: deployments.filter((target) => !stopped.includes(target.deployment)).map((t) => t.deployment),
|
|
1398
|
+
});
|
|
1399
|
+
return stopped;
|
|
1169
1400
|
},
|
|
1170
1401
|
|
|
1171
1402
|
/**
|
|
@@ -1190,31 +1421,326 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
|
|
|
1190
1421
|
},
|
|
1191
1422
|
|
|
1192
1423
|
/**
|
|
1193
|
-
* @method get-
|
|
1194
|
-
* @description
|
|
1195
|
-
*
|
|
1424
|
+
* @method get-traffic
|
|
1425
|
+
* @description Prints the live blue/green colour of every routable
|
|
1426
|
+
* deployment, of both kinds, as a table.
|
|
1427
|
+
* @param {string} [path] - Comma-separated hosts to report on; empty reports every host.
|
|
1196
1428
|
* @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
|
|
1197
1429
|
* @memberof UnderpostRun
|
|
1198
1430
|
*/
|
|
1199
|
-
'get-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1431
|
+
'get-traffic': async (path = '', options = DEFAULT_OPTION) => {
|
|
1432
|
+
options = {
|
|
1433
|
+
...options,
|
|
1434
|
+
gatewayApi: gatewayApiEnabledFactory(options),
|
|
1435
|
+
namespace: options.namespace || 'default',
|
|
1436
|
+
};
|
|
1437
|
+
// A report answers "what is live", so it cannot pick one environment from a
|
|
1438
|
+
// flag and call the other absent: a cluster running `development` would be
|
|
1439
|
+
// reported as entirely unrouted. Both are scanned and the environment is a
|
|
1440
|
+
// column; `--dev` narrows to development when only that is wanted.
|
|
1441
|
+
const envs = options.dev ? ['development'] : options.test ? ['development', 'production'] : ['production'];
|
|
1442
|
+
const hosts = `${path || ''}`
|
|
1443
|
+
.split(',')
|
|
1444
|
+
.map((host) => host.trim())
|
|
1445
|
+
.filter(Boolean);
|
|
1446
|
+
|
|
1447
|
+
const confRoot = './engine-private/conf';
|
|
1448
|
+
const deployList = options.deployId
|
|
1449
|
+
? resolveDeployList(options.deployId)
|
|
1450
|
+
: fs.existsSync(confRoot)
|
|
1451
|
+
? fs
|
|
1452
|
+
.readdirSync(confRoot)
|
|
1453
|
+
.filter((deployId) => fs.existsSync(`${confRoot}/${deployId}/conf.server.json`))
|
|
1454
|
+
.sort()
|
|
1455
|
+
: [];
|
|
1456
|
+
|
|
1457
|
+
const deployments = Underpost.kubectl.get('', 'deployment', options.namespace);
|
|
1458
|
+
const deployedNames = deployments.map((entry) => entry.NAME);
|
|
1459
|
+
|
|
1460
|
+
// Four cluster-wide reads, correlated once: which kind describes a host,
|
|
1461
|
+
// whether its listener terminates TLS, and whether QUIC is enabled on it.
|
|
1462
|
+
// A missing CRD is an empty list, so a single-stack cluster reads the same
|
|
1463
|
+
// way as one running both.
|
|
1464
|
+
const listResources = (kind) => {
|
|
1465
|
+
const raw = shellExec(`kubectl get ${kind} -A -o json`, {
|
|
1203
1466
|
stdout: true,
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
.
|
|
1209
|
-
.
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1467
|
+
silent: true,
|
|
1468
|
+
silentOnError: true,
|
|
1469
|
+
});
|
|
1470
|
+
try {
|
|
1471
|
+
const parsed = JSON.parse(`${raw || ''}`);
|
|
1472
|
+
return Array.isArray(parsed?.items) ? parsed.items : [];
|
|
1473
|
+
} catch {
|
|
1474
|
+
return [];
|
|
1475
|
+
}
|
|
1476
|
+
};
|
|
1477
|
+
const ingressFacts = hostIngressFactsFactory({
|
|
1478
|
+
httpRoutes: listResources('httproute'),
|
|
1479
|
+
httpProxies: listResources('httpproxy'),
|
|
1480
|
+
gateways: listResources('gateway'),
|
|
1481
|
+
clientTrafficPolicies: listResources('clienttrafficpolicy'),
|
|
1482
|
+
});
|
|
1483
|
+
const trafficServiceSelectors = Object.fromEntries(
|
|
1484
|
+
listResources('service')
|
|
1485
|
+
.filter((service) => service?.metadata?.labels?.['underpost.net/traffic-service'] === 'true')
|
|
1486
|
+
.map((service) => [
|
|
1487
|
+
`${service?.metadata?.namespace || 'default'}/${service?.metadata?.name}`,
|
|
1488
|
+
service?.spec?.selector?.app || '',
|
|
1489
|
+
]),
|
|
1490
|
+
);
|
|
1491
|
+
|
|
1492
|
+
// One read per host, reused across every deployment and environment that
|
|
1493
|
+
// shares it — the colour match is pure, only the fetch is expensive.
|
|
1494
|
+
const routingInfo = {};
|
|
1495
|
+
const hostRoutingInfo = (host) => {
|
|
1496
|
+
if (routingInfo[host] === undefined)
|
|
1497
|
+
routingInfo[host] = Underpost.deploy.readHostRoutingInfo({ host, options });
|
|
1498
|
+
return routingInfo[host];
|
|
1499
|
+
};
|
|
1500
|
+
const trafficState = {};
|
|
1501
|
+
const liveTrafficStateOf = (entry, env) => {
|
|
1502
|
+
const key = `${entry.id}/${env}`;
|
|
1503
|
+
if (trafficState[key]) return trafficState[key];
|
|
1504
|
+
const stableService = Underpost.deploy.trafficServiceNameFactory({ deployId: entry.id, env });
|
|
1505
|
+
const selector = trafficServiceSelectors[`${options.namespace}/${stableService}`] || '';
|
|
1506
|
+
const stableTraffic = trafficFromRoutingInfoFactory({ info: selector, deployId: entry.id, env });
|
|
1507
|
+
if (stableTraffic)
|
|
1508
|
+
return (trafficState[key] = {
|
|
1509
|
+
colour: stableTraffic,
|
|
1510
|
+
service: stableService,
|
|
1511
|
+
});
|
|
1512
|
+
const legacyTraffic = trafficFromRoutingInfoFactory({
|
|
1513
|
+
info: hostRoutingInfo(entry.host),
|
|
1514
|
+
deployId: entry.id,
|
|
1515
|
+
env,
|
|
1516
|
+
});
|
|
1517
|
+
return (trafficState[key] = {
|
|
1518
|
+
colour: legacyTraffic,
|
|
1519
|
+
service: legacyTraffic ? `${entry.deployment}-${legacyTraffic}-service` : '',
|
|
1520
|
+
});
|
|
1521
|
+
};
|
|
1522
|
+
const servingState = {};
|
|
1523
|
+
const servesTraffic = (entry, env) => {
|
|
1524
|
+
const service = liveTrafficStateOf(entry, env).service;
|
|
1525
|
+
if (!service) return false;
|
|
1526
|
+
const key = `${options.namespace}/${service}`;
|
|
1527
|
+
if (servingState[key] === undefined)
|
|
1528
|
+
servingState[key] = Underpost.deploy.serviceHasReadyEndpoints({
|
|
1529
|
+
service,
|
|
1530
|
+
namespace: options.namespace,
|
|
1531
|
+
});
|
|
1532
|
+
return servingState[key];
|
|
1533
|
+
};
|
|
1534
|
+
|
|
1535
|
+
const rows = envs.flatMap((env) =>
|
|
1536
|
+
trafficTableRowsFactory({
|
|
1537
|
+
entries: deployList
|
|
1538
|
+
.flatMap((deployId) => deployTrafficEntriesFactory({ deployId, env }))
|
|
1539
|
+
.map((entry) => ({ ...entry, env })),
|
|
1540
|
+
hosts,
|
|
1541
|
+
liveTrafficOf: (entry) => liveTrafficStateOf(entry, env).colour,
|
|
1542
|
+
servesTraffic: (entry) => servesTraffic(entry, env),
|
|
1543
|
+
}),
|
|
1213
1544
|
);
|
|
1545
|
+
|
|
1546
|
+
if (rows.length === 0) {
|
|
1547
|
+
logger.warn('No configured hosts matched the requested traffic report', {
|
|
1548
|
+
hosts,
|
|
1549
|
+
envs,
|
|
1550
|
+
deployList,
|
|
1551
|
+
namespace: options.namespace,
|
|
1552
|
+
});
|
|
1553
|
+
return rows;
|
|
1554
|
+
}
|
|
1555
|
+
|
|
1556
|
+
// Probe the exact public URL represented by each host/path row. PWA rows
|
|
1557
|
+
// can carry several configured paths in one cell, while instance rows
|
|
1558
|
+
// carry one; cache by URL so shared rows never repeat network work.
|
|
1559
|
+
const shellArg = (value) => `'${`${value}`.replaceAll("'", "'\\''")}'`;
|
|
1560
|
+
const probeCache = new Map();
|
|
1561
|
+
const probePath = (host, routePath, tls) => {
|
|
1562
|
+
const normalizedPath = `${routePath || '/'}`.startsWith('/') ? `${routePath || '/'}` : `/${routePath}`;
|
|
1563
|
+
let url;
|
|
1564
|
+
try {
|
|
1565
|
+
url = new URL(normalizedPath, `${tls ? 'https' : 'http'}://${host}`).href;
|
|
1566
|
+
} catch {
|
|
1567
|
+
return { path: normalizedPath, url: '', statuses: ['000'] };
|
|
1568
|
+
}
|
|
1569
|
+
if (!probeCache.has(url)) {
|
|
1570
|
+
// -L follows the real redirect chain, -v supplies one response line
|
|
1571
|
+
// per hop, -i keeps full response headers available, and -s removes
|
|
1572
|
+
// only the progress meter. The body is discarded to keep a status
|
|
1573
|
+
// report bounded even when a host returns a large application page.
|
|
1574
|
+
const raw = shellExec(
|
|
1575
|
+
`curl -L -v -i -s --connect-timeout 3 --max-time 12 --max-redirs 10 ` +
|
|
1576
|
+
`-o /dev/null -w '\nUNDERPOST_CURL_FINAL=%{http_code}\n' ${shellArg(url)} 2>&1 || true`,
|
|
1577
|
+
{ stdout: true, silent: true, silentOnError: true, disableLog: true },
|
|
1578
|
+
);
|
|
1579
|
+
probeCache.set(url, curlStatusChainFactory(raw));
|
|
1580
|
+
}
|
|
1581
|
+
return { path: normalizedPath, url, statuses: probeCache.get(url) };
|
|
1582
|
+
};
|
|
1583
|
+
|
|
1584
|
+
const deploymentByName = new Map(deployments.map((deployment) => [deployment.NAME, deployment]));
|
|
1585
|
+
const readinessOf = (deployment) => {
|
|
1586
|
+
const replicas = deployment?.READY || '-';
|
|
1587
|
+
const [ready, desired] = `${replicas}`.split('/').map(Number);
|
|
1588
|
+
return {
|
|
1589
|
+
replicas,
|
|
1590
|
+
exists: Boolean(deployment),
|
|
1591
|
+
ready: Number.isFinite(ready) && Number.isFinite(desired) && desired > 0 && ready === desired,
|
|
1592
|
+
};
|
|
1593
|
+
};
|
|
1594
|
+
const deploymentStatusOf = (row, traffic) => {
|
|
1595
|
+
if (!traffic) return null;
|
|
1596
|
+
const deployment = `${row.deployment}-${traffic}`;
|
|
1597
|
+
return {
|
|
1598
|
+
deployment,
|
|
1599
|
+
traffic,
|
|
1600
|
+
...readinessOf(deploymentByName.get(deployment)),
|
|
1601
|
+
};
|
|
1602
|
+
};
|
|
1603
|
+
const reportRows = rows.map((row) => {
|
|
1604
|
+
const facts = ingressFacts[row.host] || {};
|
|
1605
|
+
const probes = [...new Set(`${row.path || '/'}`.split(/\s+/).filter(Boolean))].map((routePath) =>
|
|
1606
|
+
probePath(row.host, routePath, facts.tls),
|
|
1607
|
+
);
|
|
1608
|
+
const oppositeTraffic = row.traffic ? nextTrafficFactory(row.traffic) : '';
|
|
1609
|
+
const current = deploymentStatusOf(row, row.traffic);
|
|
1610
|
+
const opposite = deploymentStatusOf(row, oppositeTraffic);
|
|
1611
|
+
return {
|
|
1612
|
+
...row,
|
|
1613
|
+
probes,
|
|
1614
|
+
current: current ? { ...current, serving: row.serving } : null,
|
|
1615
|
+
opposite,
|
|
1616
|
+
};
|
|
1617
|
+
});
|
|
1618
|
+
|
|
1619
|
+
// Padded on the raw values, coloured afterwards: an ANSI escape counts
|
|
1620
|
+
// toward String.length and would skew every column right of it.
|
|
1621
|
+
const columns = ['HOST', 'PATH', 'KIND', 'ROUTE', 'TLS', 'HTTP3', 'CURRENT', 'OPPOSITE'];
|
|
1622
|
+
const deploymentStatus = (status, includeServing = false) => {
|
|
1623
|
+
if (!status) return `unrouted - missing${includeServing ? ' not-serving' : ''}`;
|
|
1624
|
+
return [
|
|
1625
|
+
status.deployment,
|
|
1626
|
+
status.replicas,
|
|
1627
|
+
status.exists ? (status.ready ? 'ready' : 'not-ready') : 'missing',
|
|
1628
|
+
...(includeServing ? [status.serving ? 'serving' : 'not-serving'] : []),
|
|
1629
|
+
].join(' ');
|
|
1630
|
+
};
|
|
1631
|
+
const cellOf = (row) => {
|
|
1632
|
+
const facts = ingressFacts[row.host] || {};
|
|
1633
|
+
const pathStatus = row.probes.map((probe) => `${probe.path} [${probe.statuses.join('→')}]`).join(' ');
|
|
1634
|
+
return [
|
|
1635
|
+
row.host,
|
|
1636
|
+
pathStatus,
|
|
1637
|
+
row.kind,
|
|
1638
|
+
facts.route || 'none',
|
|
1639
|
+
facts.tls ? 'yes' : 'no',
|
|
1640
|
+
facts.http3 ? 'yes' : 'no',
|
|
1641
|
+
deploymentStatus(row.current, true),
|
|
1642
|
+
deploymentStatus(row.opposite),
|
|
1643
|
+
];
|
|
1644
|
+
};
|
|
1645
|
+
const paint = (value, i) => {
|
|
1646
|
+
if (columns[i] === 'PATH')
|
|
1647
|
+
return value.replace(/\b(?:000|[1-5][0-9]{2})\b/g, (status) => {
|
|
1648
|
+
if (/^1/.test(status)) return status.cyan;
|
|
1649
|
+
if (/^2/.test(status)) return status.green;
|
|
1650
|
+
if (/^3/.test(status)) return status.cyan;
|
|
1651
|
+
if (/^4/.test(status)) return status.yellow;
|
|
1652
|
+
return status.red;
|
|
1653
|
+
});
|
|
1654
|
+
if (columns[i] === 'CURRENT' || columns[i] === 'OPPOSITE') {
|
|
1655
|
+
if (value === '-') return value;
|
|
1656
|
+
const [deployment, replicas, status, serving] = value.split(' ');
|
|
1657
|
+
const deploymentDisplay = deployment.replace(
|
|
1658
|
+
/-(blue|green)$/,
|
|
1659
|
+
(_, traffic) => `-${traffic === 'blue' ? traffic.bgBlue.bold.black : traffic.bgGreen.bold.black}`,
|
|
1660
|
+
);
|
|
1661
|
+
const replicasDisplay = readinessOf({ READY: replicas }).ready ? replicas.green : replicas.red;
|
|
1662
|
+
const statusDisplay = status === 'ready' ? status.green : status.red;
|
|
1663
|
+
const servingDisplay = serving ? (serving === 'serving' ? serving.green : serving.red) : '';
|
|
1664
|
+
return [deploymentDisplay, replicasDisplay, statusDisplay, servingDisplay].filter(Boolean).join(' ');
|
|
1665
|
+
}
|
|
1666
|
+
// TLS and HTTP/3 being off is a normal development state, not a fault, so
|
|
1667
|
+
// only the affirmative is highlighted.
|
|
1668
|
+
if (columns[i] === 'TLS' || columns[i] === 'HTTP3') return value === 'yes' ? value.green : value;
|
|
1669
|
+
if (columns[i] === 'ROUTE') return value === 'none' ? value.red : value;
|
|
1670
|
+
return value;
|
|
1671
|
+
};
|
|
1672
|
+
// A shared table renderer, called once per environment when both are
|
|
1673
|
+
// scanned: mixing development and production rows into one table is what
|
|
1674
|
+
// made an unrouted production host look like a duplicate of the same,
|
|
1675
|
+
// live development host.
|
|
1676
|
+
const printTable = (cells, heading) => {
|
|
1677
|
+
const widths = columns.map((column, i) => Math.max(column.length, ...cells.map((cell) => `${cell[i]}`.length)));
|
|
1678
|
+
const line = (values, painted) =>
|
|
1679
|
+
values
|
|
1680
|
+
.map(
|
|
1681
|
+
(value, i) => (painted ? paint(`${value}`, i) : `${value}`) + ' '.repeat(widths[i] - `${value}`.length),
|
|
1682
|
+
)
|
|
1683
|
+
.join(' ');
|
|
1684
|
+
console.log(heading ? `\n${heading.bold}\n${line(columns, false).bold}` : `\n${line(columns, false).bold}`);
|
|
1685
|
+
console.log(widths.map((width) => '-'.repeat(width)).join(' '));
|
|
1686
|
+
for (const cell of cells) console.log(line(cell, true));
|
|
1687
|
+
};
|
|
1688
|
+
for (const env of envs) {
|
|
1689
|
+
const envCells = reportRows.filter((row) => row.env === env).map(cellOf);
|
|
1690
|
+
if (envCells.length > 0) printTable(envCells, `[${env.toUpperCase()}]`);
|
|
1691
|
+
}
|
|
1692
|
+
console.log('');
|
|
1693
|
+
return reportRows;
|
|
1694
|
+
},
|
|
1695
|
+
|
|
1696
|
+
/**
|
|
1697
|
+
* @method restore-mongo
|
|
1698
|
+
* @description Initializes a MongoDB replica set in the cluster without resetting existing data.
|
|
1699
|
+
* @param {string} path - The input value, identifier, or path for the operation.
|
|
1700
|
+
* @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
|
|
1701
|
+
* @memberof UnderpostRun
|
|
1702
|
+
*/
|
|
1703
|
+
'restore-mongo': async (path, options = DEFAULT_OPTION) => {
|
|
1704
|
+
await MongoBootstrap.initReplicaSet({
|
|
1705
|
+
namespace: options.namespace || 'default',
|
|
1706
|
+
reset: options.reset || false,
|
|
1707
|
+
clusterType: options.kubeadm ? 'kubeadm' : options.k3s ? 'k3s' : 'kind', // o 'k3s' / 'kubeadm' según corresponda
|
|
1708
|
+
underpostRoot: '.',
|
|
1709
|
+
});
|
|
1710
|
+
},
|
|
1711
|
+
|
|
1712
|
+
/**
|
|
1713
|
+
* @method ingress-refresh
|
|
1714
|
+
* @description Rebuilds the shared HTTPProxy/HTTPRoute host map without
|
|
1715
|
+
* inheriting application workload placement. Supplying a path or
|
|
1716
|
+
* `--ingress-node` is the explicit recovery mechanism for relocating the
|
|
1717
|
+
* public 80/443 listener.
|
|
1718
|
+
* @param {string} [path] - Optional ingress node name.
|
|
1719
|
+
* @param {UnderpostRunDefaultOptions} options - Runner options.
|
|
1720
|
+
* @returns {boolean} True after the ingress is Ready with the refreshed map.
|
|
1721
|
+
* @memberof UnderpostRun
|
|
1722
|
+
*/
|
|
1723
|
+
'ingress-refresh': (path = '', options = DEFAULT_OPTION) => {
|
|
1724
|
+
const namespace = options.namespace || 'default';
|
|
1725
|
+
const ingressNode = options.ingressNode || `${path || ''}`.trim();
|
|
1726
|
+
const updated = Underpost.cluster.refreshUnderpostIngress({
|
|
1727
|
+
namespace,
|
|
1728
|
+
options: { ...options, ingressNode },
|
|
1729
|
+
});
|
|
1730
|
+
if (!updated)
|
|
1731
|
+
throw new Error(
|
|
1732
|
+
`[ingress-refresh] ${UNDERPOST_INGRESS.name} is not installed; install both ingress stacks first`,
|
|
1733
|
+
);
|
|
1734
|
+
logger.info('[ingress-refresh] Shared ingress is operational', {
|
|
1735
|
+
namespace,
|
|
1736
|
+
node: ingressNode || '(preserved)',
|
|
1737
|
+
});
|
|
1738
|
+
return true;
|
|
1214
1739
|
},
|
|
1215
1740
|
|
|
1216
1741
|
'instance-promote': async (path, options = DEFAULT_OPTION) => {
|
|
1217
1742
|
const env = options.dev ? 'development' : 'production';
|
|
1743
|
+
options = { ...options, gatewayApi: gatewayApiEnabledFactory(options) };
|
|
1218
1744
|
let [deployId, id] = path.split(',');
|
|
1219
1745
|
const confInstances = loadConfInstances(deployId);
|
|
1220
1746
|
const promoted = selectConfInstances(confInstances, id);
|
|
@@ -1224,56 +1750,214 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
|
|
|
1224
1750
|
// host shares one object. Rebuilding it from the promoted instance alone
|
|
1225
1751
|
// would drop its siblings' routes, so each host is rendered from the full
|
|
1226
1752
|
// set: promoted instances flip colour, the rest keep their live colour.
|
|
1753
|
+
//
|
|
1754
|
+
// "Full set" is not what the conf currently declares. Each variant sub-path
|
|
1755
|
+
// is its own deployment, and one that was dropped from the conf while its
|
|
1756
|
+
// workload is still up must keep its route — otherwise editing the variant
|
|
1757
|
+
// list to scope a deploy takes the untouched variants offline. The
|
|
1758
|
+
// descriptors last published for the host cover exactly that gap, and a
|
|
1759
|
+
// variant leaves the render only once its Deployment is gone.
|
|
1227
1760
|
const promotedIds = new Set(promoted.map((instance) => instance.id));
|
|
1228
1761
|
const hosts = [...new Set(promoted.map((instance) => instance.host))];
|
|
1229
|
-
const
|
|
1762
|
+
const namespace = options.namespace || 'default';
|
|
1763
|
+
const gatewayConfDir = Underpost.deploy.gatewayConfDirFactory({ deployId, env });
|
|
1764
|
+
const deployedNames = Underpost.kubectl.get('', 'deployment', namespace).map((entry) => entry.NAME);
|
|
1765
|
+
const instancesByHost = Object.fromEntries(
|
|
1766
|
+
hosts.map((host) => [
|
|
1767
|
+
host,
|
|
1768
|
+
hostRenderInstancesFactory({
|
|
1769
|
+
declared: confInstances.filter((instance) => instance.host === host),
|
|
1770
|
+
preserved: readHostInstanceRegistry({ confDir: gatewayConfDir, host }),
|
|
1771
|
+
isDeployed: (instance) =>
|
|
1772
|
+
deployedNames.some((name) => name.startsWith(`${deployId}-${instance.id}-${env}-`)),
|
|
1773
|
+
}),
|
|
1774
|
+
]),
|
|
1775
|
+
);
|
|
1776
|
+
const affected = hosts.flatMap((host) => instancesByHost[host]);
|
|
1230
1777
|
const trafficById = {};
|
|
1778
|
+
const currentTrafficById = {};
|
|
1779
|
+
const bootstrapTrafficById = {};
|
|
1231
1780
|
for (const instance of affected) {
|
|
1232
1781
|
const currentTraffic = Underpost.deploy.getCurrentTraffic(`${deployId}-${instance.id}`, {
|
|
1233
1782
|
hostTest: instance.host,
|
|
1234
1783
|
namespace: options.namespace,
|
|
1235
1784
|
env,
|
|
1785
|
+
gatewayApi: options.gatewayApi,
|
|
1236
1786
|
});
|
|
1787
|
+
currentTrafficById[instance.id] = currentTraffic;
|
|
1237
1788
|
if (!promotedIds.has(instance.id)) {
|
|
1238
1789
|
trafficById[instance.id] = currentTraffic || 'blue';
|
|
1239
1790
|
continue;
|
|
1240
1791
|
}
|
|
1241
|
-
trafficById[instance.id] =
|
|
1792
|
+
trafficById[instance.id] = nextTrafficFactory(currentTraffic, options.targetTrafficById?.[instance.id]);
|
|
1242
1793
|
promotedTraffic = trafficById[instance.id];
|
|
1243
1794
|
}
|
|
1244
1795
|
|
|
1796
|
+
// Readiness is mandatory for an actual promotion. The only exception is
|
|
1797
|
+
// the explicit no-backend checkpoint, whose purpose is to prove that an
|
|
1798
|
+
// endpointless selector returns the configured fallback before a workload
|
|
1799
|
+
// exists.
|
|
1800
|
+
if (!options.noBackendCheckpoint)
|
|
1801
|
+
for (const instance of affected.filter((entry) => promotedIds.has(entry.id))) {
|
|
1802
|
+
const podId = `${deployId}-${instance.id}-${env}-${trafficById[instance.id]}`;
|
|
1803
|
+
if (
|
|
1804
|
+
!Underpost.deploy.awaitDeploymentReady({ deployment: podId, namespace }) ||
|
|
1805
|
+
!Underpost.deploy.awaitServiceEndpoints({ service: `${podId}-service`, namespace })
|
|
1806
|
+
)
|
|
1807
|
+
throw new Error(`Refusing to promote ${instance.id} to unready colour ${trafficById[instance.id]}`);
|
|
1808
|
+
}
|
|
1809
|
+
|
|
1810
|
+
// Bootstrap one stable Service per instance on its current ready colour.
|
|
1811
|
+
// Routes and fallback blocks can now migrate without changing traffic;
|
|
1812
|
+
// promoted selectors move to their targets only after the whole host has
|
|
1813
|
+
// converged below.
|
|
1814
|
+
for (const instance of affected) {
|
|
1815
|
+
const instanceDeployId = `${deployId}-${instance.id}`;
|
|
1816
|
+
const currentTraffic = currentTrafficById[instance.id];
|
|
1817
|
+
const currentReady =
|
|
1818
|
+
!!currentTraffic &&
|
|
1819
|
+
Underpost.deploy.serviceHasReadyEndpoints({
|
|
1820
|
+
service: `${instanceDeployId}-${env}-${currentTraffic}-service`,
|
|
1821
|
+
namespace,
|
|
1822
|
+
});
|
|
1823
|
+
const bootstrapTraffic = currentReady ? currentTraffic : trafficById[instance.id];
|
|
1824
|
+
bootstrapTrafficById[instance.id] = bootstrapTraffic;
|
|
1825
|
+
Underpost.deploy.applyTrafficService({
|
|
1826
|
+
deployId: instanceDeployId,
|
|
1827
|
+
env,
|
|
1828
|
+
traffic: bootstrapTraffic,
|
|
1829
|
+
namespace,
|
|
1830
|
+
fromPort: instancePortFactory({ instance, env }),
|
|
1831
|
+
toPort: instancePortFactory({ instance, env, container: true }),
|
|
1832
|
+
});
|
|
1833
|
+
}
|
|
1834
|
+
|
|
1835
|
+
// Instance routes attach to the Gateway the parent deploy owns. A Gateway
|
|
1836
|
+
// per instance host cannot work beside it: `mergeGateways` collapses every
|
|
1837
|
+
// Gateway of the class onto one listener per (port, protocol), so a
|
|
1838
|
+
// hostname-scoped listener and the deploy's hostname-less one contend for
|
|
1839
|
+
// the same port — on 80 the hostname-scoped one is dropped outright, and
|
|
1840
|
+
// on 443 it keeps an SNI filter chain whose route table is left empty.
|
|
1841
|
+
// Either way every path on the instance host answers 404 while the
|
|
1842
|
+
// Gateway reports Programmed and the route reports Accepted.
|
|
1843
|
+
const gatewayName = Underpost.deploy.gatewayNameFactory({ deployId, env });
|
|
1245
1844
|
for (const host of hosts) {
|
|
1246
|
-
const hostInstances =
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1845
|
+
const hostInstances = instancesByHost[host];
|
|
1846
|
+
const routingEnv = options.tls ? 'production' : env;
|
|
1847
|
+
// Every variant of this host contributes one proxied sub-path to the
|
|
1848
|
+
// shared gateway, whose interception turns each workload's own error
|
|
1849
|
+
// into that variant's declared document.
|
|
1850
|
+
if (options.gatewayApi)
|
|
1851
|
+
writeHostServerConf({
|
|
1852
|
+
confDir: gatewayConfDir,
|
|
1853
|
+
host,
|
|
1854
|
+
conf: hostServerConfFactory({
|
|
1855
|
+
host,
|
|
1856
|
+
namespace: options.namespace || 'default',
|
|
1857
|
+
routes: hostInstances.map((instance) => ({
|
|
1858
|
+
path: instance.path,
|
|
1859
|
+
upstream: `${Underpost.deploy.trafficServiceNameFactory({
|
|
1860
|
+
deployId: `${deployId}-${instance.id}`,
|
|
1861
|
+
env,
|
|
1862
|
+
})}:${instancePortFactory({ instance, env })}`,
|
|
1863
|
+
statuses: instanceInterceptStatusesFactory(instance),
|
|
1864
|
+
stripPrefix: Array.isArray(instance.pathRewritePolicy) && instance.pathRewritePolicy.length > 0,
|
|
1865
|
+
})),
|
|
1866
|
+
}),
|
|
1867
|
+
});
|
|
1868
|
+
// Recorded before the route is published, so a variant dropped from the
|
|
1869
|
+
// conf keeps its descriptor from the render that still carried it.
|
|
1870
|
+
writeHostInstanceRegistry({ confDir: gatewayConfDir, host, instances: hostInstances });
|
|
1871
|
+
// The route below sends every intercepted path to this shared Nginx
|
|
1872
|
+
// service. Load the host block first so an Accepted route can never race
|
|
1873
|
+
// a gateway that still has only its default server configuration.
|
|
1874
|
+
if (options.gatewayApi)
|
|
1875
|
+
installGatewayConf({
|
|
1876
|
+
hostRoot: Underpost.deploy.underpostGatewayRootFactory(options),
|
|
1877
|
+
confSourceDir: gatewayConfDir,
|
|
1878
|
+
namespace: options.namespace,
|
|
1879
|
+
});
|
|
1880
|
+
let proxyYaml = options.gatewayApi
|
|
1881
|
+
? Underpost.deploy.httpRouteYamlFactory({
|
|
1882
|
+
host,
|
|
1883
|
+
options,
|
|
1884
|
+
parentName: gatewayName,
|
|
1885
|
+
rules: instanceHttpRouteRulesFactory({ deployId, instances: hostInstances, env, trafficById, options }),
|
|
1886
|
+
})
|
|
1887
|
+
: Underpost.deploy.baseProxyYamlFactory({ host, env: routingEnv, options }) +
|
|
1888
|
+
instanceProxyRoutesFactory({ deployId, instances: hostInstances, env, trafficById });
|
|
1250
1889
|
if (options.tls) {
|
|
1251
1890
|
if (options.test) {
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
shellExec(
|
|
1258
|
-
`kubectl create secret tls ${host} --cert="${sslDir}/${nameSafe}.pem" --key="${sslDir}/${nameSafe}-key.pem" -n ${options.namespace}`,
|
|
1259
|
-
);
|
|
1891
|
+
Underpost.deploy.selfSignedTlsSecretFactory({
|
|
1892
|
+
host,
|
|
1893
|
+
namespace: options.namespace,
|
|
1894
|
+
underpostRoot: options.underpostRoot || '.',
|
|
1895
|
+
});
|
|
1260
1896
|
} else {
|
|
1261
1897
|
shellExec(`sudo kubectl delete Certificate ${host} -n ${options.namespace} --ignore-not-found`);
|
|
1262
1898
|
proxyYaml += Underpost.deploy.buildCertManagerCertificate({ ...options, host });
|
|
1263
1899
|
}
|
|
1264
1900
|
}
|
|
1265
|
-
|
|
1901
|
+
if (options.gatewayApi) {
|
|
1902
|
+
// Left by the per-host model this replaces. Both outlive the route
|
|
1903
|
+
// that referenced them and keep contending for the merged listener.
|
|
1904
|
+
for (const name of [`Gateway ${host}`, `ClientTrafficPolicy ${host}-http3`])
|
|
1905
|
+
shellExec(`kubectl delete ${name} --namespace ${options.namespace} --ignore-not-found`, { silent: true });
|
|
1906
|
+
}
|
|
1907
|
+
// The host's route object is replaced in place, never deleted first:
|
|
1908
|
+
// `apply` moves the whole spec in one transition, so the hostname always
|
|
1909
|
+
// has a route. Delete-then-apply left a window with none, and Envoy
|
|
1910
|
+
// answered every request that landed in it 404 — an outage on each
|
|
1911
|
+
// promote, independent of which colour was healthy.
|
|
1266
1912
|
shellExec(
|
|
1267
|
-
`kubectl apply -f - -n ${options.namespace} <<EOF
|
|
1913
|
+
`kubectl apply -f - -n ${options.namespace} <<'EOF'
|
|
1268
1914
|
${proxyYaml}
|
|
1269
1915
|
EOF
|
|
1270
1916
|
`,
|
|
1271
1917
|
{ disableLog: true },
|
|
1272
1918
|
);
|
|
1273
1919
|
}
|
|
1920
|
+
// These hostnames may have just moved between stacks, or appeared for the
|
|
1921
|
+
// first time. A shared edge that has not been told answers them from the
|
|
1922
|
+
// other data plane, which has no route for them — a 404 in front of a
|
|
1923
|
+
// healthy workload. No-op when no shared edge is installed.
|
|
1924
|
+
const sharedIngressUpdated = Underpost.cluster.refreshUnderpostIngress({ namespace, options });
|
|
1925
|
+
if (sharedIngressUpdated) {
|
|
1926
|
+
Underpost.deploy.removeInactiveHostRoutes({ hosts, gatewayApi: options.gatewayApi, namespace });
|
|
1927
|
+
Underpost.cluster.refreshUnderpostIngress({ namespace, options });
|
|
1928
|
+
}
|
|
1929
|
+
if (!options.noBackendCheckpoint)
|
|
1930
|
+
for (const instance of affected.filter((entry) => promotedIds.has(entry.id))) {
|
|
1931
|
+
const instanceDeployId = `${deployId}-${instance.id}`;
|
|
1932
|
+
const targetTraffic = trafficById[instance.id];
|
|
1933
|
+
const bootstrapTraffic = bootstrapTrafficById[instance.id];
|
|
1934
|
+
if (targetTraffic !== bootstrapTraffic)
|
|
1935
|
+
Underpost.deploy.applyTrafficService({
|
|
1936
|
+
deployId: instanceDeployId,
|
|
1937
|
+
env,
|
|
1938
|
+
traffic: targetTraffic,
|
|
1939
|
+
namespace,
|
|
1940
|
+
fromPort: instancePortFactory({ instance, env }),
|
|
1941
|
+
toPort: instancePortFactory({ instance, env, container: true }),
|
|
1942
|
+
});
|
|
1943
|
+
const trafficService = Underpost.deploy.trafficServiceNameFactory({ deployId: instanceDeployId, env });
|
|
1944
|
+
if (!Underpost.deploy.awaitServiceEndpoints({ service: trafficService, namespace })) {
|
|
1945
|
+
if (targetTraffic !== bootstrapTraffic)
|
|
1946
|
+
Underpost.deploy.applyTrafficService({
|
|
1947
|
+
deployId: instanceDeployId,
|
|
1948
|
+
env,
|
|
1949
|
+
traffic: bootstrapTraffic,
|
|
1950
|
+
namespace,
|
|
1951
|
+
fromPort: instancePortFactory({ instance, env }),
|
|
1952
|
+
toPort: instancePortFactory({ instance, env, container: true }),
|
|
1953
|
+
});
|
|
1954
|
+
throw new Error(`Traffic Service ${trafficService} never became ready on ${targetTraffic}`);
|
|
1955
|
+
}
|
|
1956
|
+
}
|
|
1274
1957
|
// Refresh the gRPC service to ensure it points to the parent deploy's current traffic.
|
|
1275
1958
|
if (promotedTraffic) {
|
|
1276
|
-
const parentTraffic =
|
|
1959
|
+
const parentTraffic =
|
|
1960
|
+
Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace, env }) || 'blue';
|
|
1277
1961
|
const grpcServicePath = Underpost.deploy.buildGrpcServiceManifest({
|
|
1278
1962
|
deployId,
|
|
1279
1963
|
env,
|
|
@@ -1293,11 +1977,75 @@ EOF
|
|
|
1293
1977
|
*/
|
|
1294
1978
|
instance: async (path = '', options = DEFAULT_OPTION) => {
|
|
1295
1979
|
const env = options.dev ? 'development' : 'production';
|
|
1980
|
+
options = {
|
|
1981
|
+
...options,
|
|
1982
|
+
gatewayApi: gatewayApiEnabledFactory(options),
|
|
1983
|
+
namespace: options.namespace || 'default',
|
|
1984
|
+
};
|
|
1296
1985
|
const baseCommand = options.dev ? 'node bin' : 'underpost';
|
|
1297
1986
|
const baseClusterCommand = options.dev ? ' --dev' : '';
|
|
1298
1987
|
let [deployId, id, replicas] = path.split(',');
|
|
1299
1988
|
if (!replicas) replicas = options.replicas;
|
|
1300
1989
|
const confInstances = selectConfInstances(loadConfInstances(deployId), id);
|
|
1990
|
+
const { liveTrafficById, targetTrafficById, serving } = instanceTrafficPlanFactory({
|
|
1991
|
+
instances: confInstances,
|
|
1992
|
+
requestedTraffic: options.traffic,
|
|
1993
|
+
liveTrafficOf: (instance) =>
|
|
1994
|
+
Underpost.deploy.getCurrentTraffic(`${deployId}-${instance.id}`, {
|
|
1995
|
+
hostTest: instance.host,
|
|
1996
|
+
namespace: options.namespace,
|
|
1997
|
+
env,
|
|
1998
|
+
gatewayApi: options.gatewayApi,
|
|
1999
|
+
}),
|
|
2000
|
+
servesTraffic: (instance, colour) =>
|
|
2001
|
+
Underpost.deploy.serviceHasReadyEndpoints({
|
|
2002
|
+
service: `${deployId}-${instance.id}-${env}-${colour}-service`,
|
|
2003
|
+
namespace: options.namespace,
|
|
2004
|
+
}),
|
|
2005
|
+
});
|
|
2006
|
+
|
|
2007
|
+
let prePromoted = false;
|
|
2008
|
+
const fallbackChecks = instanceFallbackChecksFactory(confInstances);
|
|
2009
|
+
// The promote points intercepted statuses at these documents on either
|
|
2010
|
+
// path, so they are placed regardless of which one runs.
|
|
2011
|
+
if (options.gatewayApi && fallbackChecks.length > 0 && !options.expose)
|
|
2012
|
+
placeInstanceStaticAssets({ instances: confInstances, options, label: 'instance' });
|
|
2013
|
+
if (
|
|
2014
|
+
options.gatewayApi &&
|
|
2015
|
+
!options.gatewayBootstrapComplete &&
|
|
2016
|
+
fallbackChecks.length > 0 &&
|
|
2017
|
+
!options.expose &&
|
|
2018
|
+
serving.length === 0
|
|
2019
|
+
) {
|
|
2020
|
+
// Clear the target colour before routing to it. A previous blue/green
|
|
2021
|
+
// cycle may have left that inactive Deployment Ready, which would turn
|
|
2022
|
+
// this into a stale-app probe instead of a no-backend fallback probe.
|
|
2023
|
+
for (const instance of confInstances) {
|
|
2024
|
+
const podId = `${deployId}-${instance.id}-${env}-${targetTrafficById[instance.id]}`;
|
|
2025
|
+
shellExec(`kubectl delete service ${podId}-service --namespace ${options.namespace} --ignore-not-found`);
|
|
2026
|
+
shellExec(`kubectl delete deployment ${podId} --namespace ${options.namespace} --ignore-not-found`);
|
|
2027
|
+
}
|
|
2028
|
+
// A direct `run instance` must publish and prove its configured static
|
|
2029
|
+
// fallback before it submits the first instance Deployment document.
|
|
2030
|
+
await UnderpostRun.RUNNERS['instance-promote'](`${deployId},${id}`, {
|
|
2031
|
+
...options,
|
|
2032
|
+
targetTrafficById,
|
|
2033
|
+
noBackendCheckpoint: true,
|
|
2034
|
+
});
|
|
2035
|
+
await gatewayFallbackProbeRunner({
|
|
2036
|
+
gatewayStatusRunner: UnderpostRun.RUNNERS['gateway-status'],
|
|
2037
|
+
checks: fallbackChecks,
|
|
2038
|
+
options,
|
|
2039
|
+
label: 'instance',
|
|
2040
|
+
});
|
|
2041
|
+
prePromoted = true;
|
|
2042
|
+
} else if (serving.length > 0)
|
|
2043
|
+
logger.info('[instance] Live colour serving; holding traffic until the target colour is Ready', {
|
|
2044
|
+
hosts: [...new Set(serving.map((instance) => instance.host))],
|
|
2045
|
+
live: serving.map((instance) => `${instance.id}:${liveTrafficById[instance.id]}`),
|
|
2046
|
+
target: serving.map((instance) => `${instance.id}:${targetTrafficById[instance.id]}`),
|
|
2047
|
+
});
|
|
2048
|
+
|
|
1301
2049
|
const etcHosts = [];
|
|
1302
2050
|
for (const instance of confInstances) {
|
|
1303
2051
|
let {
|
|
@@ -1305,10 +2053,6 @@ EOF
|
|
|
1305
2053
|
host: _host,
|
|
1306
2054
|
path: _path,
|
|
1307
2055
|
image: _image,
|
|
1308
|
-
fromPort: _fromPort,
|
|
1309
|
-
toPort: _toPort,
|
|
1310
|
-
fromDebugPort: _fromDebugPort,
|
|
1311
|
-
toDebugPort: _toDebugPort,
|
|
1312
2056
|
cmd: _cmd,
|
|
1313
2057
|
volumes: _volumes,
|
|
1314
2058
|
metadata: _metadata,
|
|
@@ -1317,9 +2061,8 @@ EOF
|
|
|
1317
2061
|
livenessProbe: _livenessProbe,
|
|
1318
2062
|
} = instance;
|
|
1319
2063
|
const _deployId = `${deployId}-${_id}`;
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
if (env === 'development' && _toDebugPort) _toPort = _toDebugPort;
|
|
2064
|
+
const _fromPort = instancePortFactory({ instance, env });
|
|
2065
|
+
const _toPort = instancePortFactory({ instance, env, container: true });
|
|
1323
2066
|
etcHosts.push(_host);
|
|
1324
2067
|
if (options.expose) continue;
|
|
1325
2068
|
// Examples images:
|
|
@@ -1336,13 +2079,7 @@ EOF
|
|
|
1336
2079
|
k3s: options.k3s,
|
|
1337
2080
|
});
|
|
1338
2081
|
|
|
1339
|
-
const
|
|
1340
|
-
hostTest: _host,
|
|
1341
|
-
namespace: options.namespace,
|
|
1342
|
-
env,
|
|
1343
|
-
});
|
|
1344
|
-
|
|
1345
|
-
const targetTraffic = currentTraffic ? (currentTraffic === 'blue' ? 'green' : 'blue') : 'blue';
|
|
2082
|
+
const targetTraffic = targetTrafficById[instance.id];
|
|
1346
2083
|
const podId = `${_deployId}-${env}-${targetTraffic}`;
|
|
1347
2084
|
const ignorePods = Underpost.kubectl.get(podId, 'pods', options.namespace).map((p) => p.NAME);
|
|
1348
2085
|
Underpost.deploy.configMap(env, options.namespace);
|
|
@@ -1362,20 +2099,21 @@ EOF
|
|
|
1362
2099
|
k3s: options.k3s,
|
|
1363
2100
|
env,
|
|
1364
2101
|
}),
|
|
1365
|
-
clusterContext: options
|
|
2102
|
+
clusterContext: clusterTypeFactory(options),
|
|
1366
2103
|
gitClean: options.gitClean || false,
|
|
1367
2104
|
sshKeyPath: options.sshKeyPath || '',
|
|
1368
2105
|
});
|
|
1369
2106
|
// Regenerate the parent deploy's gRPC ClusterIP service pointing to the
|
|
1370
2107
|
// parent's current traffic colour and apply it before the instance pod starts so
|
|
1371
2108
|
// DNS is resolvable the moment the pod boots.
|
|
1372
|
-
const parentTraffic =
|
|
2109
|
+
const parentTraffic =
|
|
2110
|
+
Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace, env }) || 'blue';
|
|
1373
2111
|
const grpcServicePath = Underpost.deploy.buildGrpcServiceManifest({
|
|
1374
2112
|
deployId,
|
|
1375
2113
|
env,
|
|
1376
2114
|
confServer: loadConfServerJson(`./engine-private/conf/${deployId}/conf.server.json`),
|
|
1377
2115
|
namespace: options.namespace,
|
|
1378
|
-
traffic: [
|
|
2116
|
+
traffic: [parentTraffic],
|
|
1379
2117
|
host: _host,
|
|
1380
2118
|
});
|
|
1381
2119
|
if (grpcServicePath) shellExec(`kubectl apply -f ${grpcServicePath} -n ${options.namespace}`);
|
|
@@ -1390,7 +2128,6 @@ EOF
|
|
|
1390
2128
|
// Resolve env-scoped lifecycle/probe blocks: each can be either
|
|
1391
2129
|
// { ...envObj } // shared shape
|
|
1392
2130
|
// { development: {...}, production: {...} } // env-specific
|
|
1393
|
-
const pickEnv = (v) => (v && (v.development || v.production) ? v[env] : v);
|
|
1394
2131
|
|
|
1395
2132
|
// Convention: an instance config may place `imagePullPolicy` inside
|
|
1396
2133
|
// the env-scoped lifecycle block (alongside postStart/preStop).
|
|
@@ -1398,7 +2135,7 @@ EOF
|
|
|
1398
2135
|
// strip it from the lifecycle hash so the rendered YAML stays valid.
|
|
1399
2136
|
// CLI override (`--image-pull-policy`) wins over the conf value.
|
|
1400
2137
|
const { lifecycle: lifecycleForManifest, imagePullPolicy: lifecycleImagePullPolicy } =
|
|
1401
|
-
Underpost.deploy.extractInstanceImagePullPolicy(
|
|
2138
|
+
Underpost.deploy.extractInstanceImagePullPolicy(resolveEnvScoped(_lifecycle, env));
|
|
1402
2139
|
const instanceImagePullPolicy = options.imagePullPolicy || lifecycleImagePullPolicy;
|
|
1403
2140
|
|
|
1404
2141
|
let deploymentYaml = `---
|
|
@@ -1414,16 +2151,31 @@ ${Underpost.deploy
|
|
|
1414
2151
|
volumes: _volumes,
|
|
1415
2152
|
cmd: resolvedCmd,
|
|
1416
2153
|
lifecycle: lifecycleForManifest,
|
|
1417
|
-
readinessProbe:
|
|
1418
|
-
|
|
2154
|
+
readinessProbe: Underpost.deploy.requiredReadinessProbeFactory({
|
|
2155
|
+
probe: resolveEnvScoped(_readinessProbe, env),
|
|
2156
|
+
port: _toPort,
|
|
2157
|
+
}),
|
|
2158
|
+
livenessProbe: resolveEnvScoped(_livenessProbe, env),
|
|
1419
2159
|
containerPort: _toPort,
|
|
1420
2160
|
imagePullPolicy: instanceImagePullPolicy,
|
|
2161
|
+
// Pin the pod in the manifest submitted for its only rollout. Volumes were
|
|
2162
|
+
// already resolved against this node; leaving the pod unconstrained could
|
|
2163
|
+
// schedule it away from its data and require a second, post-ready move.
|
|
2164
|
+
nodeName: options.nodeName
|
|
2165
|
+
? Underpost.deploy.resolveDeployNode({
|
|
2166
|
+
node: options.nodeName,
|
|
2167
|
+
kind: options.kind,
|
|
2168
|
+
kubeadm: options.kubeadm,
|
|
2169
|
+
k3s: options.k3s,
|
|
2170
|
+
env,
|
|
2171
|
+
})
|
|
2172
|
+
: '',
|
|
1421
2173
|
})
|
|
1422
2174
|
.replace('{{ports}}', buildKindPorts(_fromPort, _toPort))}
|
|
1423
2175
|
`;
|
|
1424
2176
|
// console.log(deploymentYaml);
|
|
1425
2177
|
shellExec(
|
|
1426
|
-
`kubectl apply -f - -n ${options.namespace} <<EOF
|
|
2178
|
+
`kubectl apply -f - -n ${options.namespace} <<'EOF'
|
|
1427
2179
|
${deploymentYaml}
|
|
1428
2180
|
EOF
|
|
1429
2181
|
`,
|
|
@@ -1446,14 +2198,12 @@ EOF
|
|
|
1446
2198
|
return;
|
|
1447
2199
|
}
|
|
1448
2200
|
}
|
|
1449
|
-
//
|
|
1450
|
-
//
|
|
1451
|
-
//
|
|
1452
|
-
//
|
|
1453
|
-
//
|
|
1454
|
-
|
|
1455
|
-
// single call with the family id promotes the family atomically.
|
|
1456
|
-
if (!options.expose) await UnderpostRun.RUNNERS['instance-promote'](`${deployId},${id}`, options);
|
|
2201
|
+
// Cluster-invoked instances inherit a fallback route for the old
|
|
2202
|
+
// colour, so they still promote the family atomically after every variant
|
|
2203
|
+
// is Ready. A direct run already routed the exact target colour before the
|
|
2204
|
+
// Deployment and proved its fallback; that route simply starts proxying the
|
|
2205
|
+
// new endpoints and must not be toggled a second time.
|
|
2206
|
+
if (!options.expose && !prePromoted) await UnderpostRun.RUNNERS['instance-promote'](`${deployId},${id}`, options);
|
|
1457
2207
|
if (options.etcHosts) {
|
|
1458
2208
|
const hostListenResult = etcHostFactory(etcHosts);
|
|
1459
2209
|
logger.info(hostListenResult.renderHosts);
|
|
@@ -1506,7 +2256,7 @@ EOF
|
|
|
1506
2256
|
* @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
|
|
1507
2257
|
* @memberof UnderpostRun
|
|
1508
2258
|
*/
|
|
1509
|
-
'instance-build-manifest': (path, options = DEFAULT_OPTION) => {
|
|
2259
|
+
'instance-build-manifest': async (path, options = DEFAULT_OPTION) => {
|
|
1510
2260
|
const env = options.dev ? 'development' : 'production';
|
|
1511
2261
|
let [deployId, id, projectPath] = path.split(',');
|
|
1512
2262
|
const rootPath = projectPath ? projectPath : '.';
|
|
@@ -1527,7 +2277,7 @@ EOF
|
|
|
1527
2277
|
}
|
|
1528
2278
|
if (!options.instanceOnly && (selected.length > 1 || selected[0].id !== id)) {
|
|
1529
2279
|
for (const instance of selected)
|
|
1530
|
-
UnderpostRun.RUNNERS['instance-build-manifest'](
|
|
2280
|
+
await UnderpostRun.RUNNERS['instance-build-manifest'](
|
|
1531
2281
|
[deployId, instance.id, projectPath].filter((v) => v !== undefined).join(','),
|
|
1532
2282
|
{ ...options, instanceOnly: true },
|
|
1533
2283
|
);
|
|
@@ -1542,15 +2292,12 @@ EOF
|
|
|
1542
2292
|
|
|
1543
2293
|
const instance = selected[0];
|
|
1544
2294
|
const isDefaultInstance = instance.id === instance.templateId || !instance.templateId;
|
|
2295
|
+
const instanceEnvBuilder = await loadProjectInstanceEnvBuilder(deployId);
|
|
1545
2296
|
|
|
1546
2297
|
let {
|
|
1547
2298
|
id: _id,
|
|
1548
2299
|
host: _host,
|
|
1549
2300
|
image: _image,
|
|
1550
|
-
fromPort: _fromPort,
|
|
1551
|
-
toPort: _toPort,
|
|
1552
|
-
fromDebugPort: _fromDebugPort,
|
|
1553
|
-
toDebugPort: _toDebugPort,
|
|
1554
2301
|
cmd: _cmd,
|
|
1555
2302
|
volumes: _volumes,
|
|
1556
2303
|
metadata: _metadata,
|
|
@@ -1587,9 +2334,8 @@ EOF
|
|
|
1587
2334
|
|
|
1588
2335
|
const _deployId = `${deployId}-${_id}`;
|
|
1589
2336
|
if (!_image) _image = `underpost/underpost-engine:${Underpost.version}`;
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
if (env === 'development' && _toDebugPort) _toPort = _toDebugPort;
|
|
2337
|
+
const _fromPort = instancePortFactory({ instance, env });
|
|
2338
|
+
const _toPort = instancePortFactory({ instance, env, container: true });
|
|
1593
2339
|
|
|
1594
2340
|
// Build image from projectPath Dockerfile and load into cluster when --build is set.
|
|
1595
2341
|
if (options.build && projectPath) {
|
|
@@ -1622,13 +2368,12 @@ EOF
|
|
|
1622
2368
|
|
|
1623
2369
|
// Env-aware lifecycle / probe selection. Each block may either be
|
|
1624
2370
|
// a single object (shared across envs) or `{ development, production }`.
|
|
1625
|
-
const pickEnv = (v) => (v && (v.development || v.production) ? v[env] : v);
|
|
1626
2371
|
|
|
1627
2372
|
// Convention: an instance config may place `imagePullPolicy` inside
|
|
1628
2373
|
// the env-scoped lifecycle block (alongside postStart/preStop).
|
|
1629
2374
|
// Extract it onto the container spec and strip it from the lifecycle hash.
|
|
1630
2375
|
const { lifecycle: lifecycleForManifest, imagePullPolicy: lifecycleImagePullPolicy } =
|
|
1631
|
-
Underpost.deploy.extractInstanceImagePullPolicy(
|
|
2376
|
+
Underpost.deploy.extractInstanceImagePullPolicy(resolveEnvScoped(_lifecycle, env));
|
|
1632
2377
|
const instanceImagePullPolicy = options.imagePullPolicy || lifecycleImagePullPolicy;
|
|
1633
2378
|
|
|
1634
2379
|
const deploymentYaml =
|
|
@@ -1645,10 +2390,22 @@ EOF
|
|
|
1645
2390
|
volumes: _volumes,
|
|
1646
2391
|
cmd: resolvedCmd,
|
|
1647
2392
|
lifecycle: lifecycleForManifest,
|
|
1648
|
-
readinessProbe:
|
|
1649
|
-
|
|
2393
|
+
readinessProbe: Underpost.deploy.requiredReadinessProbeFactory({
|
|
2394
|
+
probe: resolveEnvScoped(_readinessProbe, env),
|
|
2395
|
+
port: _toPort,
|
|
2396
|
+
}),
|
|
2397
|
+
livenessProbe: resolveEnvScoped(_livenessProbe, env),
|
|
1650
2398
|
containerPort: _toPort,
|
|
1651
2399
|
imagePullPolicy: instanceImagePullPolicy,
|
|
2400
|
+
nodeName: options.nodeName
|
|
2401
|
+
? Underpost.deploy.resolveDeployNode({
|
|
2402
|
+
node: options.nodeName,
|
|
2403
|
+
kind: options.kind,
|
|
2404
|
+
kubeadm: options.kubeadm,
|
|
2405
|
+
k3s: options.k3s,
|
|
2406
|
+
env,
|
|
2407
|
+
})
|
|
2408
|
+
: '',
|
|
1652
2409
|
})
|
|
1653
2410
|
.replace('{{ports}}', buildKindPorts(_fromPort, _toPort));
|
|
1654
2411
|
|
|
@@ -1703,6 +2460,36 @@ EOF
|
|
|
1703
2460
|
trafficById: { [instance.id]: targetTraffic },
|
|
1704
2461
|
});
|
|
1705
2462
|
|
|
2463
|
+
// A status route is only emitted for a page this project actually ships, so
|
|
2464
|
+
// a rewrite never points at a document that cannot exist. The check is a
|
|
2465
|
+
// read: placing the document into the gateway volume is `deploy
|
|
2466
|
+
// --sync-static`'s job at apply time, and a build must not mutate the host.
|
|
2467
|
+
// `projectPath` is passed through because this runner is given one
|
|
2468
|
+
// explicitly; the sync derives the same root from the instance itself.
|
|
2469
|
+
const statusPageEntries = instanceStatusPageEntriesFactory({
|
|
2470
|
+
instances: [instance],
|
|
2471
|
+
projectPath: rootPath,
|
|
2472
|
+
}).filter((entry) => fs.existsSync(entry.sourcePath));
|
|
2473
|
+
|
|
2474
|
+
// httproute.yaml — this instance's own routes, including the status routes
|
|
2475
|
+
// that reach the static utility instead of this workload. No Gateway is
|
|
2476
|
+
// emitted beside it: the parent deploy owns the one Gateway that
|
|
2477
|
+
// terminates every hostname it serves, and a second, hostname-scoped one
|
|
2478
|
+
// would contend with it for the merged listener rather than add to it.
|
|
2479
|
+
const httpRouteYaml = Underpost.deploy.httpRouteYamlFactory({
|
|
2480
|
+
host: _host,
|
|
2481
|
+
options,
|
|
2482
|
+
parentName: Underpost.deploy.gatewayNameFactory({ deployId, env }),
|
|
2483
|
+
rules: instanceHttpRouteRulesFactory({
|
|
2484
|
+
deployId,
|
|
2485
|
+
instances: [instance],
|
|
2486
|
+
env,
|
|
2487
|
+
trafficById: { [instance.id]: targetTraffic },
|
|
2488
|
+
options,
|
|
2489
|
+
servedStatuses: statusPageEntries.map((page) => `${page.status}`),
|
|
2490
|
+
}),
|
|
2491
|
+
});
|
|
2492
|
+
|
|
1706
2493
|
// grpc-service.yaml — the parent deploy's gRPC ClusterIP (shared; the
|
|
1707
2494
|
// instance cmd resolves {{grpc-service-dns}} to it). Reuse the parent's
|
|
1708
2495
|
// generated manifest when present rather than regenerating it here.
|
|
@@ -1718,27 +2505,56 @@ EOF
|
|
|
1718
2505
|
fs.writeFileSync(`${instanceBuildDir}/deployment.yaml`, deploymentYaml, 'utf8');
|
|
1719
2506
|
const siblingManifests = {
|
|
1720
2507
|
'pv-pvc.yaml': pvPvcYaml,
|
|
2508
|
+
'traffic-service.yaml': Underpost.deploy.trafficServiceYamlFactory({
|
|
2509
|
+
deployId: _deployId,
|
|
2510
|
+
env,
|
|
2511
|
+
traffic: targetTraffic,
|
|
2512
|
+
namespace: options.namespace,
|
|
2513
|
+
fromPort: _fromPort,
|
|
2514
|
+
toPort: _toPort,
|
|
2515
|
+
}),
|
|
1721
2516
|
'proxy.yaml': proxyYaml,
|
|
2517
|
+
// No gateway.yaml: the parent deploy owns the Gateway. `writeManifest`
|
|
2518
|
+
// removes the file a previous per-host build left behind.
|
|
2519
|
+
'gateway.yaml': '',
|
|
2520
|
+
'httproute.yaml': httpRouteYaml,
|
|
1722
2521
|
'grpc-service.yaml': grpcServiceYaml,
|
|
1723
2522
|
};
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
2523
|
+
// Written only when they carry objects, and removed otherwise: an empty
|
|
2524
|
+
// manifest makes `kubectl apply` fail with "no objects passed to apply",
|
|
2525
|
+
// so a build that produces none must leave none behind either.
|
|
2526
|
+
for (const [name, content] of Object.entries(siblingManifests))
|
|
2527
|
+
for (const dir of [envManifestPath, instanceBuildDir])
|
|
2528
|
+
Underpost.deploy.writeManifest({ filePath: `${dir}/${name}`, content });
|
|
1729
2529
|
logger.info('[instance-build-manifest] Sibling manifests written', {
|
|
1730
2530
|
project: envManifestPath,
|
|
1731
2531
|
enginePrivate: instanceBuildDir,
|
|
1732
2532
|
pvPvc: !!pvPvcYaml,
|
|
1733
2533
|
proxy: !!proxyYaml,
|
|
2534
|
+
httpRoute: !!httpRouteYaml,
|
|
2535
|
+
statusPages: statusPageEntries.length,
|
|
1734
2536
|
grpcService: !!grpcServiceYaml,
|
|
1735
2537
|
});
|
|
2538
|
+
const { gatewayClassName, http3, quicPort, altSvc } = Underpost.deploy.gatewayApiConfigFactory(options);
|
|
2539
|
+
logger.info('[instance-build-manifest] Gateway API manifests written', {
|
|
2540
|
+
host: _host,
|
|
2541
|
+
gatewayClass: gatewayClassName,
|
|
2542
|
+
http3,
|
|
2543
|
+
quicPort,
|
|
2544
|
+
altSvc: http3 ? altSvc : null,
|
|
2545
|
+
statusPages: (instance.customStatusPages || []).map((page) => ({
|
|
2546
|
+
status: page.status,
|
|
2547
|
+
route: `${instance.path === '/' ? '' : instance.path}/${page.status}`,
|
|
2548
|
+
hostPath: page.hostPath,
|
|
2549
|
+
})),
|
|
2550
|
+
statusPageResources: statusPageEntries,
|
|
2551
|
+
});
|
|
1736
2552
|
|
|
1737
2553
|
// --- Per-instance env files -----------------------------------------
|
|
1738
|
-
// Each env file
|
|
1739
|
-
// same mode
|
|
1740
|
-
//
|
|
1741
|
-
//
|
|
2554
|
+
// Each env file starts from the template instance's canonical file for the
|
|
2555
|
+
// same mode. Operator-owned keys remain private and are copied verbatim;
|
|
2556
|
+
// deploy-specific builders may then derive application env from the
|
|
2557
|
+
// normalized instance path/code.
|
|
1742
2558
|
//
|
|
1743
2559
|
// A derived instance's env dir is generated in full: both development.env
|
|
1744
2560
|
// and production.env are written on every build, so a deploy in either
|
|
@@ -1751,18 +2567,24 @@ EOF
|
|
|
1751
2567
|
const envsToWrite = isDefaultInstance ? [env] : ['development', 'production'];
|
|
1752
2568
|
for (const targetEnv of envsToWrite) {
|
|
1753
2569
|
const templateEnvPath = `./engine-private/conf/${deployId}/instances/${instance.templateId}/env/${targetEnv}.env`;
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
2570
|
+
if (!fs.existsSync(templateEnvPath))
|
|
2571
|
+
throw new Error(`[instance-build-manifest] Missing canonical env file: ${templateEnvPath}`);
|
|
2572
|
+
const baseEnv = dotenv.parse(fs.readFileSync(templateEnvPath, 'utf8'));
|
|
2573
|
+
const builtEnv = dispatchBuildInstanceEnv({
|
|
2574
|
+
deployId,
|
|
2575
|
+
instance,
|
|
2576
|
+
environment: targetEnv,
|
|
2577
|
+
baseEnv,
|
|
2578
|
+
containerDeployId: `${_deployId}-${targetEnv}`,
|
|
2579
|
+
builders: instanceEnvBuilder ? { [deployId]: instanceEnvBuilder } : {},
|
|
1759
2580
|
});
|
|
2581
|
+
writeEnv(`${instanceEnvDir}/${targetEnv}.env`, builtEnv);
|
|
1760
2582
|
}
|
|
1761
2583
|
logger.info('[instance-build-manifest] Instance env written', {
|
|
1762
2584
|
dir: instanceEnvDir,
|
|
1763
2585
|
instanceCode: instance.instanceCode,
|
|
1764
2586
|
envs: envsToWrite,
|
|
1765
|
-
|
|
2587
|
+
builder: instanceEnvBuilder?.name || 'canonical-copy',
|
|
1766
2588
|
});
|
|
1767
2589
|
}
|
|
1768
2590
|
|
|
@@ -1772,9 +2594,19 @@ EOF
|
|
|
1772
2594
|
}
|
|
1773
2595
|
fs.copyFileSync(outputPath, `${rootPath}/deployment.yaml`);
|
|
1774
2596
|
// Sibling manifests alongside deployment.yaml at the project root.
|
|
1775
|
-
for (const name of [
|
|
2597
|
+
for (const name of [
|
|
2598
|
+
'pv-pvc.yaml',
|
|
2599
|
+
'traffic-service.yaml',
|
|
2600
|
+
'proxy.yaml',
|
|
2601
|
+
'gateway.yaml',
|
|
2602
|
+
'httproute.yaml',
|
|
2603
|
+
'grpc-service.yaml',
|
|
2604
|
+
]) {
|
|
1776
2605
|
const src = `${envManifestPath}/${name}`;
|
|
2606
|
+
// Absence is mirrored too, so the repo never ships a manifest this
|
|
2607
|
+
// build stopped producing.
|
|
1777
2608
|
if (fs.existsSync(src)) fs.copyFileSync(src, `${rootPath}/${name}`);
|
|
2609
|
+
else fs.removeSync(`${rootPath}/${name}`);
|
|
1778
2610
|
}
|
|
1779
2611
|
logger.info('[instance-build-manifest] Production artifacts copied to project root', {
|
|
1780
2612
|
rootPath,
|
|
@@ -1981,9 +2813,7 @@ EOF`);
|
|
|
1981
2813
|
shellExec(`kubectl apply -k ${underpostRoot}/manifests/deployment/adminer/. -n ${options.namespace}`);
|
|
1982
2814
|
const successInstance = await Underpost.test.statusMonitor('adminer', 'Running', 'pods', 1000, 60 * 10);
|
|
1983
2815
|
|
|
1984
|
-
if (successInstance)
|
|
1985
|
-
shellExec(`underpost deploy --expose adminer --namespace ${options.namespace}`);
|
|
1986
|
-
}
|
|
2816
|
+
if (successInstance) return UnderpostRun.RUNNERS.expose(path || 'adminer', options);
|
|
1987
2817
|
},
|
|
1988
2818
|
|
|
1989
2819
|
/**
|
|
@@ -2042,6 +2872,7 @@ EOF`);
|
|
|
2042
2872
|
* @memberof UnderpostRun
|
|
2043
2873
|
*/
|
|
2044
2874
|
promote: async (path, options = DEFAULT_OPTION) => {
|
|
2875
|
+
options = { ...options, gatewayApi: gatewayApiEnabledFactory(options) };
|
|
2045
2876
|
let [inputDeployId, inputEnv, inputReplicas] = path.split(',');
|
|
2046
2877
|
if (!inputEnv) inputEnv = 'production';
|
|
2047
2878
|
if (!inputReplicas) inputReplicas = 1;
|
|
@@ -2069,13 +2900,19 @@ EOF`);
|
|
|
2069
2900
|
|
|
2070
2901
|
if (inputDeployId === 'dd') {
|
|
2071
2902
|
for (const deployId of fs.readFileSync(`./engine-private/deploy/dd.router`, 'utf8').split(',')) {
|
|
2072
|
-
const currentTraffic = Underpost.deploy.getCurrentTraffic(deployId, {
|
|
2903
|
+
const currentTraffic = Underpost.deploy.getCurrentTraffic(deployId, {
|
|
2904
|
+
namespace: options.namespace,
|
|
2905
|
+
env: inputEnv,
|
|
2906
|
+
});
|
|
2073
2907
|
const targetTraffic = currentTraffic === 'blue' ? 'green' : 'blue';
|
|
2074
2908
|
Underpost.deploy.switchTraffic(deployId, inputEnv, targetTraffic, inputReplicas, options.namespace, options);
|
|
2075
2909
|
applyCerts(deployId, targetTraffic);
|
|
2076
2910
|
}
|
|
2077
2911
|
} else {
|
|
2078
|
-
const currentTraffic = Underpost.deploy.getCurrentTraffic(inputDeployId, {
|
|
2912
|
+
const currentTraffic = Underpost.deploy.getCurrentTraffic(inputDeployId, {
|
|
2913
|
+
namespace: options.namespace,
|
|
2914
|
+
env: inputEnv,
|
|
2915
|
+
});
|
|
2079
2916
|
const targetTraffic = currentTraffic === 'blue' ? 'green' : 'blue';
|
|
2080
2917
|
Underpost.deploy.switchTraffic(
|
|
2081
2918
|
inputDeployId,
|
|
@@ -2128,8 +2965,24 @@ EOF`);
|
|
|
2128
2965
|
},
|
|
2129
2966
|
/**
|
|
2130
2967
|
* @method cluster
|
|
2131
|
-
* @description Deploys a full production/development ready Kubernetes cluster environment including MongoDB,
|
|
2132
|
-
*
|
|
2968
|
+
* @description Deploys a full production/development ready Kubernetes cluster environment including MongoDB,
|
|
2969
|
+
* MariaDB, Valkey, the Gateway API data plane (Envoy Gateway), Contour, and Cert-Manager, and deploys all services.
|
|
2970
|
+
*
|
|
2971
|
+
* Ingress is served by the Gateway API stack (Gateway + HTTPRoute) with QUIC/HTTP3 in **both** environments;
|
|
2972
|
+
* `--disable-gateway-api` falls back to the Contour HTTPProxy stack. Because HTTP/3 has no cleartext transport,
|
|
2973
|
+
* development terminates TLS too: a self-signed certificate per host (mkcert via `scripts/ssl.sh`, whose root CA
|
|
2974
|
+
* the script installs into the system + NSS trust stores) is issued into the secret the Gateway listener
|
|
2975
|
+
* references, and every host is written to `/etc/hosts` so the operator's browser reaches the PWA at
|
|
2976
|
+
* `https://<host>` on the local machine.
|
|
2977
|
+
*
|
|
2978
|
+
* Custom instances are the optional third segment of `path`. They are resolved per deploy against that deploy's
|
|
2979
|
+
* own `conf.instances.json`, so an id only runs where its deploy declares it, and each one is deployed after its
|
|
2980
|
+
* deploy's default workload is serving — an instance reads the parent's world configuration over the parent's
|
|
2981
|
+
* gRPC ClusterIP at boot. Instance hosts share the deploy's environment: the same self-signed certificates and
|
|
2982
|
+
* `/etc/hosts` pass in development, the same cert-manager issuance in production.
|
|
2983
|
+
* @param {string} path - `<runtime-image>,<deploy-list>[,<instance-list>]` — `+`-separated lists, e.g.
|
|
2984
|
+
* `express,dd-cyberia,mmo-server` or `express,dd-cyberia+dd-core,mmo-server+mmo-client`. An instance list entry
|
|
2985
|
+
* may be a template id (`mmo-server`), which selects its whole variant family.
|
|
2133
2986
|
* @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
|
|
2134
2987
|
* @memberof UnderpostRun
|
|
2135
2988
|
*/
|
|
@@ -2138,30 +2991,39 @@ EOF`);
|
|
|
2138
2991
|
const env = options.dev ? 'development' : 'production';
|
|
2139
2992
|
const baseCommand = options.dev ? 'node bin' : 'underpost';
|
|
2140
2993
|
const baseClusterCommand = options.dev ? ' --dev' : '';
|
|
2141
|
-
const clusterType = options
|
|
2994
|
+
const clusterType = clusterTypeFactory(options, 'kubeadm');
|
|
2142
2995
|
shellCd(`/home/dd/engine`);
|
|
2143
2996
|
shellExec(`${baseCommand} cluster${baseClusterCommand} --reset --${clusterType}`);
|
|
2144
2997
|
await timer(5000);
|
|
2145
2998
|
shellExec(`${baseCommand} cluster${baseClusterCommand} --${clusterType}`);
|
|
2146
2999
|
await timer(5000);
|
|
2147
|
-
let [runtimeImage, deployList] =
|
|
3000
|
+
let [runtimeImage, deployList, instanceListId] =
|
|
2148
3001
|
path && path.trim() && path.split(',')
|
|
2149
3002
|
? path.split(',')
|
|
2150
3003
|
: [
|
|
2151
3004
|
'express',
|
|
2152
3005
|
fs.readFileSync(`${underpostRoot}/engine-private/deploy/dd.router`, 'utf8').replaceAll(',', '+'),
|
|
3006
|
+
'',
|
|
2153
3007
|
];
|
|
2154
|
-
shellExec(
|
|
2155
|
-
|
|
2156
|
-
|
|
2157
|
-
|
|
2158
|
-
);
|
|
3008
|
+
// shellExec(
|
|
3009
|
+
// `${baseCommand} image${baseClusterCommand} --build ${
|
|
3010
|
+
// runtimeImage ? ` --pull-base --path ${underpostRoot}/src/runtime/${runtimeImage}` : ''
|
|
3011
|
+
// } --${clusterType}`,
|
|
3012
|
+
// );
|
|
2159
3013
|
if (!deployList) {
|
|
2160
3014
|
deployList = [];
|
|
2161
3015
|
logger.warn('No deploy list provided');
|
|
2162
3016
|
} else deployList = deployList.split('+');
|
|
2163
3017
|
await timer(5000);
|
|
2164
|
-
|
|
3018
|
+
// --reset-mongodb wipes the retained hostPath volumes before the rollout.
|
|
3019
|
+
// This workflow already tore the node down and re-imports every database
|
|
3020
|
+
// from its git backup a few lines below, so inheriting the previous
|
|
3021
|
+
// cluster's replica set config is never wanted — that state leaves mongod
|
|
3022
|
+
// parked outside its own config and the bootstrap fighting to recover it.
|
|
3023
|
+
shellExec(`${baseCommand} cluster${baseClusterCommand} --${clusterType} --pull-image --mongodb --reset-mongodb`);
|
|
3024
|
+
if (deployList.includes('dd-cyberia'))
|
|
3025
|
+
shellExec(`${baseCommand} cluster${baseClusterCommand} --${clusterType} --pull-image --ipfs --replicas 1`);
|
|
3026
|
+
|
|
2165
3027
|
if (runtimeImage === 'lampp') {
|
|
2166
3028
|
await timer(5000);
|
|
2167
3029
|
shellExec(`${baseCommand} cluster${baseClusterCommand} --${clusterType} --pull-image --mariadb`);
|
|
@@ -2174,19 +3036,614 @@ EOF`);
|
|
|
2174
3036
|
}
|
|
2175
3037
|
await timer(5000);
|
|
2176
3038
|
shellExec(`${baseCommand} cluster${baseClusterCommand} --${clusterType} --pull-image --valkey`);
|
|
3039
|
+
// Exactly one ingress stack is installed, because the two cannot coexist
|
|
3040
|
+
// on this node: Contour's Envoy DaemonSet declares hostPort 80/443, so the
|
|
3041
|
+
// CNI hostport plugin DNATs everything arriving on those ports straight to
|
|
3042
|
+
// it — before the Gateway API data plane's own listener can see them. With
|
|
3043
|
+
// no HTTPProxy objects to program (this workflow applies HTTPRoutes),
|
|
3044
|
+
// Contour's Envoy has no listeners and refuses the redirected connection,
|
|
3045
|
+
// which looks exactly like a gateway that is not listening at all.
|
|
3046
|
+
const gatewayApi = gatewayApiEnabledFactory(options);
|
|
3047
|
+
const gatewayApiFlags = Underpost.deploy.gatewayApiFlagsFactory({ ...options, gatewayApi });
|
|
2177
3048
|
await timer(5000);
|
|
2178
|
-
|
|
3049
|
+
if (gatewayApi) {
|
|
3050
|
+
shellExec(
|
|
3051
|
+
`${baseCommand} cluster${baseClusterCommand} --${clusterType} --gateway-api${
|
|
3052
|
+
options.gatewayClass ? ` --gateway-class ${options.gatewayClass}` : ''
|
|
3053
|
+
}`,
|
|
3054
|
+
);
|
|
3055
|
+
} else shellExec(`${baseCommand} cluster${baseClusterCommand} --${clusterType} --contour`);
|
|
3056
|
+
if (gatewayApi)
|
|
3057
|
+
shellExec(
|
|
3058
|
+
`kubectl rollout status deployment/${UNDERPOST_GATEWAY.name} -n ${options.namespace || 'default'} --timeout=5m`,
|
|
3059
|
+
);
|
|
2179
3060
|
if (env === 'production') {
|
|
2180
3061
|
await timer(5000);
|
|
2181
3062
|
shellExec(`${baseCommand} cluster${baseClusterCommand} --${clusterType} --cert-manager`);
|
|
2182
3063
|
}
|
|
3064
|
+
|
|
3065
|
+
const { byDeployId: instancesByDeployId, unmatched: unmatchedInstanceIds } = clusterInstancesFactory(
|
|
3066
|
+
deployList,
|
|
3067
|
+
instanceListId,
|
|
3068
|
+
);
|
|
3069
|
+
if (unmatchedInstanceIds.length > 0)
|
|
3070
|
+
logger.warn('[cluster] No deploy declares these instances; they will not be deployed', {
|
|
3071
|
+
instances: unmatchedInstanceIds,
|
|
3072
|
+
deployList,
|
|
3073
|
+
});
|
|
3074
|
+
|
|
3075
|
+
// Development terminates TLS with a locally trusted certificate instead of
|
|
3076
|
+
// cert-manager: QUIC/HTTP3 has no cleartext transport, so without it the
|
|
3077
|
+
// dev gateway would fall back to an HTTP-only listener. The hosts are
|
|
3078
|
+
// written to /etc/hosts in a single pass — etcHostFactory rewrites the
|
|
3079
|
+
// file, so one call per deploy would drop the previous deploy's entries.
|
|
3080
|
+
// Instance hosts come through the same resolver the Gateway's certificate
|
|
3081
|
+
// list uses, so the two can never disagree about what the deploy serves.
|
|
3082
|
+
const hosts = [...new Set(deployList.flatMap((deployId) => deployHostsFactory(deployId)))];
|
|
3083
|
+
if (env === 'development') {
|
|
3084
|
+
for (const host of hosts)
|
|
3085
|
+
Underpost.deploy.selfSignedTlsSecretFactory({
|
|
3086
|
+
host,
|
|
3087
|
+
namespace: options.namespace || 'default',
|
|
3088
|
+
underpostRoot,
|
|
3089
|
+
});
|
|
3090
|
+
const hostListenResult = etcHostFactory(hosts);
|
|
3091
|
+
logger.info(hostListenResult.renderHosts);
|
|
3092
|
+
}
|
|
3093
|
+
const version = 'v3.2.90';
|
|
3094
|
+
const instanceOptionsFactory = (deployId, instanceId) => ({
|
|
3095
|
+
...options,
|
|
3096
|
+
...clusterContextFactory(clusterType),
|
|
3097
|
+
gatewayApi,
|
|
3098
|
+
gatewayBootstrapComplete: true,
|
|
3099
|
+
tls: true,
|
|
3100
|
+
test: env === 'development',
|
|
3101
|
+
etcHosts: false,
|
|
3102
|
+
namespace: options.namespace || 'default',
|
|
3103
|
+
imageName:
|
|
3104
|
+
deployId === 'dd-cyberia' && env === 'development' && instanceId === 'mmo-server'
|
|
3105
|
+
? `underpost/cyberia-server-dev:${version}`
|
|
3106
|
+
: deployId === 'dd-cyberia' && env === 'development' && instanceId === 'mmo-client'
|
|
3107
|
+
? `underpost/cyberia-client-dev:${version}`
|
|
3108
|
+
: undefined,
|
|
3109
|
+
});
|
|
3110
|
+
const deployFlagsById = {};
|
|
3111
|
+
const fallbackChecks = new Map();
|
|
3112
|
+
|
|
3113
|
+
// Regenerating the manifests is required, not incidental: the TLS listener
|
|
3114
|
+
// — and with it the QUIC policy and the HTTPRoute set — is only emitted
|
|
3115
|
+
// when the TLS and gateway flags are known at generation time. It takes two
|
|
3116
|
+
// passes because `--build-manifest` returns after writing the manifests, so
|
|
3117
|
+
// the same flags have to be repeated on the apply call.
|
|
2183
3118
|
for (const deployId of deployList) {
|
|
2184
|
-
|
|
2185
|
-
|
|
2186
|
-
|
|
2187
|
-
|
|
3119
|
+
const deployFlags =
|
|
3120
|
+
`--${clusterType}${env === 'production' ? ' --cert' : ' --self-signed'}${gatewayApiFlags}` +
|
|
3121
|
+
`${options.namespace ? ` --namespace ${options.namespace}` : ''}` +
|
|
3122
|
+
(deployId === 'dd-cyberia'
|
|
3123
|
+
? ` --image 'underpost/engine-cyberia:${version}' \
|
|
3124
|
+
--versions blue \
|
|
3125
|
+
--image-pull-policy Always \
|
|
3126
|
+
--cmd 'cd /home/dd/engine, \
|
|
3127
|
+
underpost clone underpostnet/engine-cyberia, \
|
|
3128
|
+
mkdir -p /home/dd/engine/src/client/public/itemledger \
|
|
3129
|
+
/home/dd/engine/src/client/public/cryptokoyn \
|
|
3130
|
+
/home/dd/engine/src/client/components/cryptokoyn \
|
|
3131
|
+
/home/dd/engine/src/client/components/itemledger \
|
|
3132
|
+
/home/dd/engine/hardhat, \
|
|
3133
|
+
cp -a ./engine-cyberia/src/client/public/itemledger/. /home/dd/engine/src/client/public/itemledger/, \
|
|
3134
|
+
cp -a ./engine-cyberia/src/client/public/cryptokoyn/. /home/dd/engine/src/client/public/cryptokoyn/, \
|
|
3135
|
+
cp -a ./engine-cyberia/src/client/components/cryptokoyn/. /home/dd/engine/src/client/components/cryptokoyn/, \
|
|
3136
|
+
cp -a ./engine-cyberia/src/client/components/itemledger/. /home/dd/engine/src/client/components/itemledger/, \
|
|
3137
|
+
cp -a ./engine-cyberia/src/client/Itemledger.index.js /home/dd/engine/src/client/Itemledger.index.js, \
|
|
3138
|
+
cp -a ./engine-cyberia/src/client/Cryptokoyn.index.js /home/dd/engine/src/client/Cryptokoyn.index.js, \
|
|
3139
|
+
rm -rf ./engine-cyberia, \
|
|
3140
|
+
sudo rm -rf ./engine-private/, \
|
|
3141
|
+
node bin clone underpostnet/engine-cyberia-private, \
|
|
3142
|
+
sudo mv ./engine-cyberia-private ./engine-private, \
|
|
3143
|
+
node bin env dd-cyberia ${env}, \
|
|
3144
|
+
node ./engine-private/itc-scripts/dd-cyberia-0.js, \
|
|
3145
|
+
sudo chown -R dd:dd /home/dd/engine/src/client/public/cyberia, \
|
|
3146
|
+
node bin env dd-cyberia ${env}, \
|
|
3147
|
+
node bin client dd-cyberia ${env}, \
|
|
3148
|
+
node bin start dd-cyberia ${env} --run'`
|
|
3149
|
+
: '');
|
|
3150
|
+
deployFlagsById[deployId] = deployFlags;
|
|
3151
|
+
// SSR status and context documents belong to the ingress bootstrap, so
|
|
3152
|
+
// build them on the host before any workload Deployment is submitted.
|
|
3153
|
+
shellExec(`${baseCommand} client ${deployId} ${env}`);
|
|
3154
|
+
shellExec(`${baseCommand} deploy ${deployId} ${env} --build-manifest ${deployFlags}`);
|
|
3155
|
+
// Seed the static tree before the routes exist, so every status page and
|
|
3156
|
+
// intercepted context the manifests just pointed at resolves from the
|
|
3157
|
+
// first request. This pass places what this checkout built; the pass
|
|
3158
|
+
// after the rollout replaces each document with the container's own,
|
|
3159
|
+
// which is the only place clients built from private sources exist.
|
|
3160
|
+
if (gatewayApi) {
|
|
3161
|
+
const staticAssets = Underpost.deploy.syncStaticAssets(deployId, env, {
|
|
3162
|
+
...options,
|
|
3163
|
+
...clusterContextFactory(clusterType),
|
|
3164
|
+
gatewayApi,
|
|
3165
|
+
namespace: options.namespace || 'default',
|
|
3166
|
+
versions: /--versions\s+([^\s]+)/.exec(deployFlags)?.[1] || options.versions || 'blue',
|
|
3167
|
+
});
|
|
3168
|
+
const missingAssets = staticAssets.filter((entry) => !entry.source);
|
|
3169
|
+
if (missingAssets.length > 0)
|
|
3170
|
+
throw new Error(
|
|
3171
|
+
`[cluster] Static gateway bootstrap is missing configured assets for ${deployId}: ` +
|
|
3172
|
+
missingAssets.map((entry) => entry.assetPath).join(', '),
|
|
3173
|
+
);
|
|
3174
|
+
|
|
3175
|
+
// Record the exact documents the no-backend checkpoint must return.
|
|
3176
|
+
// PWA paths use the SSR maintenance view; custom instances have no
|
|
3177
|
+
// maintenance view and reuse their first declared status document.
|
|
3178
|
+
const confServer = loadConfServerJson(`./engine-private/conf/${deployId}/conf.server.json`);
|
|
3179
|
+
const confSSRPath = `./engine-private/conf/${deployId}/conf.ssr.json`;
|
|
3180
|
+
const confSSR = fs.existsSync(confSSRPath) ? JSON.parse(fs.readFileSync(confSSRPath, 'utf8')) : {};
|
|
3181
|
+
for (const host of Object.keys(confServer))
|
|
3182
|
+
for (const path of Object.keys(confServer[host])) {
|
|
3183
|
+
const maintenance = Underpost.deploy
|
|
3184
|
+
.edgeRouteEntriesFactory({ confServer, confSSR, host, path })
|
|
3185
|
+
.find((entry) => entry.context === 'maintenance');
|
|
3186
|
+
if (maintenance)
|
|
3187
|
+
fallbackChecks.set(`${host}${path}`, {
|
|
3188
|
+
host,
|
|
3189
|
+
path,
|
|
3190
|
+
assetPath: maintenance.assetPath,
|
|
3191
|
+
kind: maintenance.kind,
|
|
3192
|
+
});
|
|
3193
|
+
}
|
|
3194
|
+
const selectedInstances = instancesByDeployId[deployId].ids.flatMap((instanceId) =>
|
|
3195
|
+
selectConfInstances(loadConfInstances(deployId), instanceId),
|
|
3196
|
+
);
|
|
3197
|
+
for (const entry of instanceStatusPageEntriesFactory({ instances: selectedInstances }))
|
|
3198
|
+
if (!fallbackChecks.has(`${entry.host}${entry.path}`))
|
|
3199
|
+
fallbackChecks.set(`${entry.host}${entry.path}`, {
|
|
3200
|
+
host: entry.host,
|
|
3201
|
+
path: entry.path,
|
|
3202
|
+
assetPath: entry.assetPath,
|
|
3203
|
+
kind: `status:${entry.status}`,
|
|
3204
|
+
});
|
|
3205
|
+
}
|
|
3206
|
+
// Apply only the gateway tier. Application Services and Deployments are
|
|
3207
|
+
// deliberately absent so the status fallback can be observed first.
|
|
3208
|
+
shellExec(`${baseCommand} deploy ${deployId} ${env} --disable-update-deployment ${deployFlags}`);
|
|
3209
|
+
// Instance host routes and their custom status pages are part of
|
|
3210
|
+
// the same ingress bootstrap. `instance-promote` is safe here: it only
|
|
3211
|
+
// writes the Nginx host block and routing objects; no instance Deployment
|
|
3212
|
+
// is created until the second phase below.
|
|
3213
|
+
for (const instanceId of instancesByDeployId[deployId].ids) {
|
|
3214
|
+
logger.info('[cluster] Bootstrapping custom instance gateway', { deployId, instanceId, env });
|
|
3215
|
+
await UnderpostRun.RUNNERS['instance-promote'](`${deployId},${instanceId}`, {
|
|
3216
|
+
...instanceOptionsFactory(deployId, instanceId),
|
|
3217
|
+
noBackendCheckpoint: true,
|
|
3218
|
+
});
|
|
3219
|
+
}
|
|
3220
|
+
}
|
|
3221
|
+
|
|
3222
|
+
// This is the deliberate no-backend checkpoint. The static Nginx pod, all
|
|
3223
|
+
// Gateway listeners, parent HTTPRoutes and selected instance HTTPRoutes
|
|
3224
|
+
// must be live before the first application Deployment YAML is submitted.
|
|
3225
|
+
if (gatewayApi && fallbackChecks.size > 0) {
|
|
3226
|
+
const fallbackResults = await gatewayFallbackProbeRunner({
|
|
3227
|
+
gatewayStatusRunner: UnderpostRun.RUNNERS['gateway-status'],
|
|
3228
|
+
checks: [...fallbackChecks.values()],
|
|
3229
|
+
options: { ...options, gatewayApi, namespace: options.namespace || 'default' },
|
|
3230
|
+
label: 'cluster',
|
|
3231
|
+
});
|
|
3232
|
+
logger.info('[cluster] Gateway fallback checkpoint passed; starting application deployments', {
|
|
3233
|
+
hosts,
|
|
3234
|
+
fallbacks: fallbackResults,
|
|
3235
|
+
});
|
|
3236
|
+
}
|
|
3237
|
+
|
|
3238
|
+
for (const deployId of deployList) {
|
|
3239
|
+
const deployFlags = deployFlagsById[deployId];
|
|
3240
|
+
// Preserve the already-operational ingress objects and apply only the
|
|
3241
|
+
// workload manifests. EndpointSlices will update as pods become Ready;
|
|
3242
|
+
// the site route continues to reach underpost-gateway throughout.
|
|
3243
|
+
shellExec(`${baseCommand} deploy ${deployId} ${env} --disable-update-proxy ${deployFlags}`);
|
|
3244
|
+
if (gatewayApi) {
|
|
3245
|
+
const namespace = options.namespace || 'default';
|
|
3246
|
+
const version = /--versions\s+([^\s,]+)/.exec(deployFlags)?.[1] || 'blue';
|
|
3247
|
+
shellExec(`kubectl rollout status deployment/${deployId}-${env}-${version} -n ${namespace} --timeout=15m`);
|
|
3248
|
+
shellExec(`${baseCommand} deploy ${deployId} ${env} --sync-static ${deployFlags}`);
|
|
3249
|
+
}
|
|
3250
|
+
|
|
3251
|
+
// Custom instance pods depend on the parent's gRPC service, so they are
|
|
3252
|
+
// still started after the parent is Ready. Their routes already exist and
|
|
3253
|
+
// keep serving the custom fallback until the atomic promotion completes.
|
|
3254
|
+
for (const instanceId of instancesByDeployId[deployId].ids) {
|
|
3255
|
+
logger.info('[cluster] Deploying custom instance', { deployId, instanceId, env, clusterType });
|
|
3256
|
+
await UnderpostRun.RUNNERS.instance(
|
|
3257
|
+
`${deployId},${instanceId},${options.replicas || 1}`,
|
|
3258
|
+
instanceOptionsFactory(deployId, instanceId),
|
|
3259
|
+
);
|
|
3260
|
+
}
|
|
3261
|
+
}
|
|
3262
|
+
logger.info('[cluster] Ingress stack deployed', {
|
|
3263
|
+
env,
|
|
3264
|
+
stack: gatewayApi ? 'gateway-api' : 'httpproxy',
|
|
3265
|
+
gatewayClass: gatewayApi ? Underpost.deploy.gatewayApiConfigFactory(options).gatewayClassName : null,
|
|
3266
|
+
http3: gatewayApi && options.disableHttp3 !== true,
|
|
3267
|
+
tls: env === 'production' ? 'cert-manager' : 'self-signed',
|
|
3268
|
+
hosts,
|
|
3269
|
+
instances: Object.fromEntries(
|
|
3270
|
+
deployList
|
|
3271
|
+
.filter((deployId) => instancesByDeployId[deployId].ids.length > 0)
|
|
3272
|
+
.map((deployId) => [deployId, instancesByDeployId[deployId].ids]),
|
|
3273
|
+
),
|
|
3274
|
+
});
|
|
3275
|
+
if (gatewayApi) await UnderpostRun.RUNNERS['gateway-status']('', options);
|
|
3276
|
+
},
|
|
3277
|
+
|
|
3278
|
+
/**
|
|
3279
|
+
* @method gateway-status
|
|
3280
|
+
* @description Reports whether the Gateway API data plane is actually
|
|
3281
|
+
* serving. Applying a Gateway only records intent: the controller
|
|
3282
|
+
* provisions Envoy asynchronously, so a deploy can finish cleanly while
|
|
3283
|
+
* nothing listens on the node — the failure then surfaces much later as a
|
|
3284
|
+
* bare "connection refused" from the browser. This waits for the Gateways to
|
|
3285
|
+
* be Programmed and prints the data plane's pods and services.
|
|
3286
|
+
* @param {string} path - Unused.
|
|
3287
|
+
* @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
|
|
3288
|
+
* @memberof UnderpostRun
|
|
3289
|
+
*/
|
|
3290
|
+
'gateway-status': async (path = '', options = DEFAULT_OPTION) => {
|
|
3291
|
+
const namespace = options.namespace || 'default';
|
|
3292
|
+
const dataPlaneNamespace = 'envoy-gateway-system';
|
|
3293
|
+
const capture = (cmd) => shellExec(cmd, { stdout: true, silent: true, silentOnError: true })?.trim?.() || '';
|
|
3294
|
+
|
|
3295
|
+
const programmed = shellExec(
|
|
3296
|
+
`kubectl wait --for=condition=Programmed --timeout=180s gateway --all -n ${namespace}`,
|
|
3297
|
+
{ silentOnError: true },
|
|
3298
|
+
);
|
|
3299
|
+
if (programmed?.code !== 0)
|
|
3300
|
+
logger.warn(
|
|
3301
|
+
'[gateway-status] Gateways are not Programmed. The listeners are not being served; ' +
|
|
3302
|
+
`check 'kubectl describe gateway -n ${namespace}' and the controller logs in ${dataPlaneNamespace}.`,
|
|
2188
3303
|
);
|
|
3304
|
+
shellExec(`kubectl get gateway -n ${namespace} -o wide`, { silentOnError: true });
|
|
3305
|
+
|
|
3306
|
+
// Per-listener truth. A Gateway reports Programmed while individual
|
|
3307
|
+
// listeners are rejected, so the aggregate condition hides exactly the
|
|
3308
|
+
// case where some hostnames serve and others do not. Prints each
|
|
3309
|
+
// listener's attached route count and any failing condition with its
|
|
3310
|
+
// reason — an unresolved TLS secret or a rejected listener names itself.
|
|
3311
|
+
const listenerStatus = capture(
|
|
3312
|
+
`kubectl get gateway -n ${namespace} -o jsonpath=` +
|
|
3313
|
+
`'{range .items[*]}{.metadata.name}{" "}` +
|
|
3314
|
+
`{range .status.listeners[*]}{.name}=routes:{.attachedRoutes}` +
|
|
3315
|
+
`{range .conditions[?(@.status=="False")]}{" "}{.type}/{.reason}{end}{" "}{end}{"\\n"}{end}'`,
|
|
3316
|
+
);
|
|
3317
|
+
logger.info('[gateway-status] Listeners (name=routes + failing conditions)\n' + (listenerStatus || '(none)'));
|
|
3318
|
+
|
|
3319
|
+
// Envoy Gateway policies attach to a Gateway but configure the merged
|
|
3320
|
+
// listener, so two of them competing for the same listener is resolved by
|
|
3321
|
+
// rejecting one — and the rejection is recorded here, not on the Gateway.
|
|
3322
|
+
const policyStatus = capture(
|
|
3323
|
+
`kubectl get clienttrafficpolicy -n ${namespace} -o jsonpath=` +
|
|
3324
|
+
`'{range .items[*]}{.metadata.name}{range .status.ancestors[*]}` +
|
|
3325
|
+
`{range .conditions[?(@.status=="False")]}{" "}{.type}/{.reason}: {.message}{end}{end}{"\\n"}{end}'`,
|
|
3326
|
+
);
|
|
3327
|
+
logger.info(
|
|
3328
|
+
'[gateway-status] ClientTrafficPolicies (failing conditions)\n' +
|
|
3329
|
+
(policyStatus
|
|
3330
|
+
.split('\n')
|
|
3331
|
+
.filter((line) => line.includes(' '))
|
|
3332
|
+
.join('\n') || '(none — all policies accepted)'),
|
|
3333
|
+
);
|
|
3334
|
+
|
|
3335
|
+
// Envoy Gateway policies that fail to translate are not visible on the
|
|
3336
|
+
// Gateway or the route: the resource is admitted, its status carries the
|
|
3337
|
+
// rejection, and the xDS snapshot it belonged to can go with it — which
|
|
3338
|
+
// reads downstream as every hostname answering route_not_found.
|
|
3339
|
+
const backendPolicyStatus = capture(
|
|
3340
|
+
`kubectl get backendtrafficpolicy -n ${namespace} -o jsonpath=` +
|
|
3341
|
+
`'{range .items[*]}{.metadata.name}{range .status.ancestors[*]}` +
|
|
3342
|
+
`{range .conditions[?(@.status=="False")]}{" "}{.type}/{.reason}: {.message}{end}{end}{"\n"}{end}' ` +
|
|
3343
|
+
`2>/dev/null`,
|
|
3344
|
+
);
|
|
3345
|
+
logger.info(
|
|
3346
|
+
'[gateway-status] BackendTrafficPolicies (failing conditions)\n' +
|
|
3347
|
+
(backendPolicyStatus
|
|
3348
|
+
.split('\n')
|
|
3349
|
+
.filter((line) => line.includes(' '))
|
|
3350
|
+
.join('\n') || '(none — all policies accepted)'),
|
|
3351
|
+
);
|
|
3352
|
+
|
|
3353
|
+
// Route-level conditions. A rule that references something unresolvable
|
|
3354
|
+
// can cost the whole route, and then every path on that hostname answers
|
|
3355
|
+
// with a bare gateway 404 while the Gateway and its listeners stay green.
|
|
3356
|
+
const routeStatus = capture(
|
|
3357
|
+
`kubectl get httproute -n ${namespace} -o jsonpath=` +
|
|
3358
|
+
`'{range .items[*]}{.metadata.name}{range .status.parents[*]}` +
|
|
3359
|
+
`{range .conditions[?(@.status=="False")]}{" "}{.type}/{.reason}: {.message}{end}{end}{"\\n"}{end}'`,
|
|
3360
|
+
)
|
|
3361
|
+
.split('\n')
|
|
3362
|
+
.filter((line) => line.includes(' '));
|
|
3363
|
+
logger.info(
|
|
3364
|
+
'[gateway-status] HTTPRoutes (failing conditions)\n' +
|
|
3365
|
+
(routeStatus.join('\n') || '(none — all routes accepted)'),
|
|
3366
|
+
);
|
|
3367
|
+
|
|
3368
|
+
// The workloads the routes point at. A status code that moves between runs
|
|
3369
|
+
// (500 here, 404 there) is the application's, not the gateway's, and the
|
|
3370
|
+
// gateway config cannot explain it — restarts and unready containers can.
|
|
3371
|
+
const backends = capture(
|
|
3372
|
+
`kubectl get pods -n ${namespace} -o custom-columns=` +
|
|
3373
|
+
`'NAME:.metadata.name,READY:.status.containerStatuses[*].ready,RESTARTS:.status.containerStatuses[*].restartCount,STATUS:.status.phase'`,
|
|
3374
|
+
);
|
|
3375
|
+
logger.info('[gateway-status] Workloads behind the routes\n' + (backends || '(none)'));
|
|
3376
|
+
|
|
3377
|
+
// A Programmed Gateway only means Envoy was provisioned — not that it is
|
|
3378
|
+
// reachable from this machine. Which of the two is false decides the fix,
|
|
3379
|
+
// so report the pod's network mode and container ports, the service type
|
|
3380
|
+
// and node ports, and what the host is actually listening on.
|
|
3381
|
+
const dataPlane = capture(
|
|
3382
|
+
`kubectl get pods -n ${dataPlaneNamespace} -o custom-columns=` +
|
|
3383
|
+
`'NAME:.metadata.name,HOST_NETWORK:.spec.hostNetwork,PORTS:.spec.containers[*].ports[*].containerPort'`,
|
|
3384
|
+
);
|
|
3385
|
+
const services = capture(
|
|
3386
|
+
`kubectl get svc -n ${dataPlaneNamespace} -o custom-columns=` +
|
|
3387
|
+
`'NAME:.metadata.name,TYPE:.spec.type,PORTS:.spec.ports[*].port,NODEPORTS:.spec.ports[*].nodePort,TARGETS:.spec.ports[*].targetPort'`,
|
|
3388
|
+
);
|
|
3389
|
+
logger.info('[gateway-status] Data plane\n' + dataPlane + '\n\n' + services);
|
|
3390
|
+
|
|
3391
|
+
// Envoy creates its listener sockets only once the control plane has
|
|
3392
|
+
// pushed a config with routes attached, which lands a little after the
|
|
3393
|
+
// pod reports Ready. Polling that window is the difference between
|
|
3394
|
+
// "connection refused" and a working gateway, so wait for the socket
|
|
3395
|
+
// rather than sampling it once.
|
|
3396
|
+
const listenerFilter = `grep -E ':(80|443|10080|10443) '`;
|
|
3397
|
+
let hostListeners = '';
|
|
3398
|
+
for (let attempt = 0; attempt < 30; attempt++) {
|
|
3399
|
+
hostListeners = capture(`sudo ss -lntupH 2>/dev/null | ${listenerFilter}`);
|
|
3400
|
+
if (/:443\s/.test(hostListeners)) break;
|
|
3401
|
+
await timer(2000);
|
|
2189
3402
|
}
|
|
3403
|
+
const servesHttps = /:443\s/.test(hostListeners);
|
|
3404
|
+
logger.info('[gateway-status] Host listeners on 80/443/10080/10443\n' + (hostListeners || '(none)'));
|
|
3405
|
+
|
|
3406
|
+
if (!servesHttps) {
|
|
3407
|
+
logger.warn(
|
|
3408
|
+
'[gateway-status] Nothing is listening on this host port 443, so a browser reaching the ' +
|
|
3409
|
+
'hostnames through /etc/hosts gets "connection refused". Compare the two tables above: ' +
|
|
3410
|
+
'HOST_NETWORK=false means the pod is on the pod network (only the ClusterIP/NodePort is ' +
|
|
3411
|
+
'reachable); HOST_NETWORK=true with PORTS 10080/10443 means the privileged-port remap is ' +
|
|
3412
|
+
'still active. Until it is resolved, forward the merged service to expose it locally:\n' +
|
|
3413
|
+
` kubectl port-forward -n ${dataPlaneNamespace} svc/<envoy-service> 443:443 80:80 --address 127.0.0.1`,
|
|
3414
|
+
);
|
|
3415
|
+
return { programmed: programmed?.code === 0, servesHttps, dataPlane, services, hostListeners, probes: [] };
|
|
3416
|
+
}
|
|
3417
|
+
|
|
3418
|
+
// An open socket still is not proof the hostname routes anywhere. Probe
|
|
3419
|
+
// each Gateway hostname exactly as the browser would — through /etc/hosts,
|
|
3420
|
+
// validating the certificate against the trust store `scripts/ssl.sh`
|
|
3421
|
+
// populated — so the workflow ends on an observed response, not an
|
|
3422
|
+
// inference. Each host is probed twice: over loopback, and pinned to the
|
|
3423
|
+
// node IP. The two answers separate a loopback-specific block from a data
|
|
3424
|
+
// plane that is not reachable at all.
|
|
3425
|
+
const nodeIp = capture(
|
|
3426
|
+
`kubectl get node -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}'`,
|
|
3427
|
+
);
|
|
3428
|
+
// The `server` header is reported alongside the status because the code
|
|
3429
|
+
// alone cannot say who produced it: `500 (envoy)` is a gateway or upstream
|
|
3430
|
+
// failure, while `500` from the workload's own server is an application
|
|
3431
|
+
// error, and the two lead to completely different fixes.
|
|
3432
|
+
const probeUrl = (url, resolveArgs = '') => {
|
|
3433
|
+
const raw = capture(
|
|
3434
|
+
`curl -sS -o /dev/null -D - -w 'HTTP_CODE=%{http_code}' --max-time 10 ${resolveArgs}${url} 2>&1 | tr -d '\\r'`,
|
|
3435
|
+
);
|
|
3436
|
+
const status = /HTTP_CODE=([0-9]{3})/.exec(raw)?.[1];
|
|
3437
|
+
if (!status) return raw.split('\n').find((line) => line.startsWith('curl:')) || 'no-response';
|
|
3438
|
+
const server = /^server:\s*(.+)$/im.exec(raw)?.[1]?.trim();
|
|
3439
|
+
return server ? `${status} (${server})` : status;
|
|
3440
|
+
};
|
|
3441
|
+
const probe = (host, resolveTo) =>
|
|
3442
|
+
probeUrl(`https://${host}`, resolveTo ? `--resolve ${host}:443:${resolveTo} ` : '');
|
|
3443
|
+
// Port 80 is the same Envoy process, same host, same listener set — only
|
|
3444
|
+
// the port differs. It separates "this gateway is unreachable" from
|
|
3445
|
+
// "something specifically rejects 443".
|
|
3446
|
+
const probeHttp = (host) => probeUrl(`http://${host}`);
|
|
3447
|
+
// curl writes `000` when it never got a response, so a bare three-digit
|
|
3448
|
+
// match would read a failed connection as success.
|
|
3449
|
+
const answered = (status) => /^[1-5][0-9]{2}\b/.test(status);
|
|
3450
|
+
// The routes, not the listeners, are where the hostnames live: a
|
|
3451
|
+
// consolidated Gateway serves every hostname from one hostname-less
|
|
3452
|
+
// listener and picks the certificate by SNI, so reading the listeners
|
|
3453
|
+
// would leave nothing to probe.
|
|
3454
|
+
const hosts = [
|
|
3455
|
+
...new Set(
|
|
3456
|
+
(path
|
|
3457
|
+
? path.split(',')
|
|
3458
|
+
: capture(`kubectl get httproute -n ${namespace} -o jsonpath='{.items[*].spec.hostnames[*]}'`).split(/\s+/)
|
|
3459
|
+
)
|
|
3460
|
+
.map((host) => host.trim())
|
|
3461
|
+
.filter(Boolean),
|
|
3462
|
+
),
|
|
3463
|
+
];
|
|
3464
|
+
const probes = hosts.map((host) => ({
|
|
3465
|
+
host,
|
|
3466
|
+
http: probeHttp(host),
|
|
3467
|
+
loopback: probe(host),
|
|
3468
|
+
...(nodeIp ? { nodeIp: probe(host, nodeIp) } : {}),
|
|
3469
|
+
}));
|
|
3470
|
+
logger.info('[gateway-status] HTTPS probe', { nodeIp: nodeIp || '(unknown)', probes });
|
|
3471
|
+
|
|
3472
|
+
// A gateway answer alone cannot say whether the gateway or the workload
|
|
3473
|
+
// produced the code: Envoy relays an upstream response unchanged. So when
|
|
3474
|
+
// a hostname fails, ask its backend the *same* question from inside the
|
|
3475
|
+
// cluster, bypassing Envoy.
|
|
3476
|
+
//
|
|
3477
|
+
// Same question is the whole point: these workloads route by virtual host,
|
|
3478
|
+
// so a probe carrying `Host: <service-name>` exercises a different branch
|
|
3479
|
+
// than the browser did and its answer means nothing. Each probe therefore
|
|
3480
|
+
// replays the route it came from — the Gateway hostname, the rule path,
|
|
3481
|
+
// and the rewrite the rule would have applied — against the rule's own
|
|
3482
|
+
// backend. Only then do the two columns compare.
|
|
3483
|
+
const failing = probes.filter((entry) => /^[45]/.test(entry.loopback) || /^[45]/.test(entry.http));
|
|
3484
|
+
if (failing.length > 0) {
|
|
3485
|
+
const routes = JSON.parse(
|
|
3486
|
+
capture(`kubectl get httproute -n ${namespace} -o json`) || '{"items":[]}',
|
|
3487
|
+
).items.filter((route) => failing.some((entry) => route.spec?.hostnames?.includes(entry.host)));
|
|
3488
|
+
// Exec into the static utility rather than spawning a probe pod: it is
|
|
3489
|
+
// installed with the gateway stack, sits in this namespace, and its
|
|
3490
|
+
// BusyBox shell already carries wget — no image pull, no pod churn.
|
|
3491
|
+
const probePod = capture(
|
|
3492
|
+
`kubectl get pods -n ${namespace} -l app=${UNDERPOST_GATEWAY.name} -o jsonpath='{.items[0].metadata.name}'`,
|
|
3493
|
+
);
|
|
3494
|
+
if (!probePod || routes.length === 0) {
|
|
3495
|
+
logger.warn(
|
|
3496
|
+
'[gateway-status] Cannot probe backends directly: ' +
|
|
3497
|
+
(probePod
|
|
3498
|
+
? 'no HTTPRoute matched a failing hostname'
|
|
3499
|
+
: `no ${UNDERPOST_GATEWAY.name} pod in ${namespace}`),
|
|
3500
|
+
);
|
|
3501
|
+
} else {
|
|
3502
|
+
const backendProbes = [];
|
|
3503
|
+
for (const route of routes) {
|
|
3504
|
+
const host = route.spec.hostnames[0];
|
|
3505
|
+
for (const rule of route.spec.rules || []) {
|
|
3506
|
+
const backend = (rule.backendRefs || [])[0];
|
|
3507
|
+
if (!backend) continue;
|
|
3508
|
+
const match = rule.matches?.[0]?.path?.value || '/';
|
|
3509
|
+
const rewrite = (rule.filters || []).find((filter) => filter.type === 'URLRewrite')?.urlRewrite?.path;
|
|
3510
|
+
const target =
|
|
3511
|
+
rewrite?.type === 'ReplaceFullPath'
|
|
3512
|
+
? rewrite.replaceFullPath
|
|
3513
|
+
: rewrite?.type === 'ReplacePrefixMatch'
|
|
3514
|
+
? rewrite.replacePrefixMatch
|
|
3515
|
+
: match;
|
|
3516
|
+
const raw = capture(
|
|
3517
|
+
`kubectl exec -n ${namespace} ${probePod} -- wget -S -O /dev/null -T 5 ` +
|
|
3518
|
+
`--header 'Host: ${host}' http://${backend.name}:${backend.port}${target} 2>&1 || true`,
|
|
3519
|
+
);
|
|
3520
|
+
const code = /HTTP\/[0-9.]+\s+([0-9]{3})/.exec(raw)?.[1];
|
|
3521
|
+
const failure = raw.split('\n').find((line) => line.includes('wget:'));
|
|
3522
|
+
backendProbes.push({
|
|
3523
|
+
request: `${host}${match}`,
|
|
3524
|
+
backend: `${backend.name}:${backend.port}${target}`,
|
|
3525
|
+
direct: code || failure?.trim() || 'no-response',
|
|
3526
|
+
gateway: failing.find((entry) => entry.host === host)?.http,
|
|
3527
|
+
});
|
|
3528
|
+
}
|
|
3529
|
+
}
|
|
3530
|
+
logger.info('[gateway-status] Backend probe (same Host and path, bypassing Envoy)', { backendProbes });
|
|
3531
|
+
// Only the rule the browser actually hit is comparable, so judge on
|
|
3532
|
+
// the root rule rather than on every rule of the route.
|
|
3533
|
+
const rootProbes = backendProbes.filter((entry) => entry.request.endsWith('/'));
|
|
3534
|
+
const appFault = rootProbes.filter((entry) => `${entry.direct}` === `${entry.gateway}`);
|
|
3535
|
+
const gatewayFault = rootProbes.filter(
|
|
3536
|
+
(entry) => /^[0-9]{3}$/.test(entry.direct) && `${entry.direct}` !== `${entry.gateway}`,
|
|
3537
|
+
);
|
|
3538
|
+
if (gatewayFault.length > 0) {
|
|
3539
|
+
logger.warn(
|
|
3540
|
+
'[gateway-status] These backends answer differently without Envoy in the path, so the gateway is ' +
|
|
3541
|
+
'not relaying the workload response — the routing layer is the fault:\n ' +
|
|
3542
|
+
gatewayFault
|
|
3543
|
+
.map((entry) => `${entry.request} -> backend ${entry.direct}, gateway ${entry.gateway}`)
|
|
3544
|
+
.join('\n '),
|
|
3545
|
+
);
|
|
3546
|
+
// The access log is the only artifact that says *why*. Its
|
|
3547
|
+
// response_flags column separates an Envoy local reply (NR no route,
|
|
3548
|
+
// UF upstream connect failure, UH no healthy upstream, DPE protocol
|
|
3549
|
+
// error) from a relayed upstream status, which no amount of probing
|
|
3550
|
+
// from outside can distinguish.
|
|
3551
|
+
// Selector, not owning-gateway labels: under `mergeGateways` the data
|
|
3552
|
+
// plane belongs to the GatewayClass, so per-Gateway labels are absent
|
|
3553
|
+
// and a selector built from them silently matches nothing.
|
|
3554
|
+
const dataPlaneLog = capture(
|
|
3555
|
+
`kubectl logs -n ${dataPlaneNamespace} ` +
|
|
3556
|
+
`-l app.kubernetes.io/name=envoy,app.kubernetes.io/component=proxy ` +
|
|
3557
|
+
`-c envoy --tail=60 --prefix 2>/dev/null | tail -60`,
|
|
3558
|
+
);
|
|
3559
|
+
logger.info(
|
|
3560
|
+
'[gateway-status] Data plane access log (response_flags names the reason)\n' +
|
|
3561
|
+
(dataPlaneLog || `(none — check: kubectl get pods -n ${dataPlaneNamespace} --show-labels)`),
|
|
3562
|
+
);
|
|
3563
|
+
// A translation the control plane rejected after admitting the
|
|
3564
|
+
// resource surfaces only here, never on the object's own status.
|
|
3565
|
+
const controlPlaneLog = capture(
|
|
3566
|
+
`kubectl logs -n ${dataPlaneNamespace} deployment/envoy-gateway --tail=40 2>/dev/null ` +
|
|
3567
|
+
`| grep -Ei 'error|warn|reject|invalid' | tail -20`,
|
|
3568
|
+
);
|
|
3569
|
+
logger.info('[gateway-status] Control plane errors\n' + (controlPlaneLog || '(none in the last 40 lines)'));
|
|
3570
|
+
}
|
|
3571
|
+
if (appFault.length > 0)
|
|
3572
|
+
logger.warn(
|
|
3573
|
+
'[gateway-status] These backends return the same code without Envoy in the path, so the gateway is ' +
|
|
3574
|
+
'relaying an application response — the routing layer is not the fault:\n ' +
|
|
3575
|
+
appFault.map((entry) => `${entry.request} -> ${entry.direct}`).join('\n ') +
|
|
3576
|
+
`\n Read the workload log: kubectl logs -n ${namespace} <pod>`,
|
|
3577
|
+
);
|
|
3578
|
+
}
|
|
3579
|
+
}
|
|
3580
|
+
|
|
3581
|
+
const unreachable = probes.filter((entry) => !answered(entry.loopback));
|
|
3582
|
+
if (unreachable.length > 0) {
|
|
3583
|
+
// Narrow it here rather than leaving the operator with a bare refusal.
|
|
3584
|
+
// A socket bound on 0.0.0.0 that resets a loopback connection is either
|
|
3585
|
+
// not in this network namespace, or something is rejecting the packet.
|
|
3586
|
+
const listenerPid = /pid=(\d+)/.exec(hostListeners)?.[1];
|
|
3587
|
+
const hostNetns = capture(`sudo readlink /proc/1/ns/net`);
|
|
3588
|
+
const listenerNetns = listenerPid ? capture(`sudo readlink /proc/${listenerPid}/ns/net`) : '';
|
|
3589
|
+
// Anything that can answer a SYN with ICMP port-unreachable, from either
|
|
3590
|
+
// rule engine. firewalld on RHEL 9 keeps its rules in its own `inet
|
|
3591
|
+
// firewalld` nftables table, which `iptables-save` cannot see at all —
|
|
3592
|
+
// grepping only iptables hides half the packet path.
|
|
3593
|
+
const rejectRules = capture(
|
|
3594
|
+
`sudo iptables-save 2>/dev/null | grep -E '(REJECT|DROP)' | grep -E '(dport|dpt:) ?(443|https)\\b' | head -20`,
|
|
3595
|
+
);
|
|
3596
|
+
const nftRules = capture(
|
|
3597
|
+
`sudo nft list ruleset 2>/dev/null | grep -nE '(dport|ports) .*(443|https)|reject' | head -20`,
|
|
3598
|
+
);
|
|
3599
|
+
// A hostPort claim outranks any process listening on the node: the CNI
|
|
3600
|
+
// plugin DNATs the port to the claiming pod first, so the packet never
|
|
3601
|
+
// reaches the gateway. This is the one failure that leaves every other
|
|
3602
|
+
// signal green.
|
|
3603
|
+
const hostPortHijack = capture(
|
|
3604
|
+
`sudo nft list ruleset 2>/dev/null | grep -E 'dport (80|443)\\b.*dnat to' | head -5`,
|
|
3605
|
+
);
|
|
3606
|
+
const hostPortClaims = capture(
|
|
3607
|
+
`kubectl get daemonset,deployment -A ` +
|
|
3608
|
+
`-o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name}={.spec.template.spec.containers[*].ports[*].hostPort}{"\\n"}{end}'`,
|
|
3609
|
+
)
|
|
3610
|
+
.split('\n')
|
|
3611
|
+
.filter((line) => /=.*\b(80|443)\b/.test(line));
|
|
3612
|
+
logger.warn(`[gateway-status] ${unreachable.length}/${probes.length} hostnames did not answer over HTTPS`, {
|
|
3613
|
+
listenerPid: listenerPid || '(unknown)',
|
|
3614
|
+
hostNetns: hostNetns || '(unknown)',
|
|
3615
|
+
listenerNetns: listenerNetns || '(unknown)',
|
|
3616
|
+
sameNetworkNamespace: !!hostNetns && hostNetns === listenerNetns,
|
|
3617
|
+
nodeIpReachable: probes.some((entry) => answered(entry.nodeIp)),
|
|
3618
|
+
endpointlessServices: capture(
|
|
3619
|
+
`kubectl get svc -A -o jsonpath=` + `'{range .items[*]}{.metadata.namespace}/{.metadata.name} {end}'`,
|
|
3620
|
+
)
|
|
3621
|
+
.split(/\s+/)
|
|
3622
|
+
.filter(Boolean)
|
|
3623
|
+
.filter((ref) => {
|
|
3624
|
+
const [ns, name] = ref.split('/');
|
|
3625
|
+
return !capture(
|
|
3626
|
+
`kubectl get endpointslice -n ${ns} -l kubernetes.io/service-name=${name} ` +
|
|
3627
|
+
`-o jsonpath='{.items[*].endpoints[*].addresses[*]}'`,
|
|
3628
|
+
).trim();
|
|
3629
|
+
}),
|
|
3630
|
+
httpReachable: probes.some((entry) => answered(entry.http)),
|
|
3631
|
+
hostPortHijack: hostPortHijack || '(none)',
|
|
3632
|
+
hostPortClaims: hostPortClaims.length > 0 ? hostPortClaims : '(none)',
|
|
3633
|
+
rejectRules: rejectRules || '(none matching 443 in iptables)',
|
|
3634
|
+
nftRules: nftRules || '(none matching 443 in nftables)',
|
|
3635
|
+
});
|
|
3636
|
+
logger.warn(
|
|
3637
|
+
'[gateway-status] Read it as: hostPortHijack non-empty → another workload reserved host port ' +
|
|
3638
|
+
'80/443 and the CNI DNATs those ports to it before the gateway can see them; hostPortClaims ' +
|
|
3639
|
+
'names the owner, and only one ingress stack can hold them. httpReachable=true → the same Envoy ' +
|
|
3640
|
+
'answers on 80, so only 443 is rejected. sameNetworkNamespace=false → the listener is not on ' +
|
|
3641
|
+
'this host despite hostNetwork. endpointlessServices naming a Service that publishes 80/443 → ' +
|
|
3642
|
+
'kube-proxy REJECTs those ports on its behalf; delete it.',
|
|
3643
|
+
);
|
|
3644
|
+
}
|
|
3645
|
+
|
|
3646
|
+
return { programmed: programmed?.code === 0, servesHttps, dataPlane, services, hostListeners, probes };
|
|
2190
3647
|
},
|
|
2191
3648
|
/**
|
|
2192
3649
|
* @method deploy
|
|
@@ -2197,11 +3654,11 @@ EOF`);
|
|
|
2197
3654
|
*/
|
|
2198
3655
|
deploy: async (path, options = DEFAULT_OPTION) => {
|
|
2199
3656
|
const deployId = path;
|
|
3657
|
+
const env = options.dev ? 'development' : 'production';
|
|
2200
3658
|
const { validVersion } = Underpost.repo.privateConfUpdate(deployId);
|
|
2201
3659
|
if (!validVersion) throw new Error('Version mismatch');
|
|
2202
|
-
const currentTraffic = Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace });
|
|
3660
|
+
const currentTraffic = Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace, env });
|
|
2203
3661
|
const targetTraffic = currentTraffic === 'blue' ? 'green' : 'blue';
|
|
2204
|
-
const env = options.dev ? 'development' : 'production';
|
|
2205
3662
|
const ignorePods = Underpost.kubectl
|
|
2206
3663
|
.get(`${deployId}-${env}-${targetTraffic}`, 'pods', options.namespace)
|
|
2207
3664
|
.map((p) => p.NAME);
|
|
@@ -2388,7 +3845,11 @@ EOF`);
|
|
|
2388
3845
|
}
|
|
2389
3846
|
const success = await Underpost.test.statusMonitor(podToMonitor);
|
|
2390
3847
|
if (success) {
|
|
2391
|
-
const versions =
|
|
3848
|
+
const versions =
|
|
3849
|
+
Underpost.deploy.getCurrentTraffic(deployId, {
|
|
3850
|
+
namespace: options.namespace,
|
|
3851
|
+
env: options.dev ? 'development' : 'production',
|
|
3852
|
+
}) || 'blue';
|
|
2392
3853
|
if (!node) node = os.hostname();
|
|
2393
3854
|
const timeoutFlags = Underpost.deploy.timeoutFlagsFactory(options);
|
|
2394
3855
|
shellExec(
|
|
@@ -2419,10 +3880,7 @@ EOF`);
|
|
|
2419
3880
|
*/
|
|
2420
3881
|
'etc-hosts': async (path = '', options = DEFAULT_OPTION) => {
|
|
2421
3882
|
const hosts = path ? path.split(',') : [];
|
|
2422
|
-
if (options.deployId)
|
|
2423
|
-
const confServer = loadConfServerJson(`./engine-private/conf/${options.deployId}/conf.server.json`);
|
|
2424
|
-
hosts.push(...Object.keys(confServer));
|
|
2425
|
-
}
|
|
3883
|
+
if (options.deployId) hosts.push(...deployHostsFactory(options.deployId));
|
|
2426
3884
|
const hostListenResult = etcHostFactory(hosts);
|
|
2427
3885
|
logger.info(hostListenResult.renderHosts);
|
|
2428
3886
|
},
|
|
@@ -2775,31 +4233,314 @@ EOF`);
|
|
|
2775
4233
|
* @memberof UnderpostRun
|
|
2776
4234
|
*/
|
|
2777
4235
|
'generate-pass': (path, options = DEFAULT_OPTION) => {
|
|
2778
|
-
const
|
|
2779
|
-
const lower = 'abcdefghijklmnopqrstuvwxyz';
|
|
2780
|
-
const upper = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ';
|
|
2781
|
-
const digits = '0123456789';
|
|
2782
|
-
const special = '@#$%^&*()_+';
|
|
2783
|
-
const all = lower + upper + digits + special;
|
|
2784
|
-
const buf = crypto.randomBytes(length + 4);
|
|
2785
|
-
// Guarantee at least one character from each required class
|
|
2786
|
-
const chars = [
|
|
2787
|
-
lower[buf[0] % lower.length],
|
|
2788
|
-
upper[buf[1] % upper.length],
|
|
2789
|
-
digits[buf[2] % digits.length],
|
|
2790
|
-
special[buf[3] % special.length],
|
|
2791
|
-
];
|
|
2792
|
-
for (let i = 4; i < length; i++) chars.push(all[buf[i] % all.length]);
|
|
2793
|
-
// Fisher-Yates shuffle using an independent random buffer
|
|
2794
|
-
const shuf = crypto.randomBytes(length);
|
|
2795
|
-
for (let i = chars.length - 1; i > 0; i--) {
|
|
2796
|
-
const j = shuf[i % shuf.length] % (i + 1);
|
|
2797
|
-
[chars[i], chars[j]] = [chars[j], chars[i]];
|
|
2798
|
-
}
|
|
2799
|
-
const password = chars.join('');
|
|
4236
|
+
const password = generateSecurePassword(path && parseInt(path) > 0 ? parseInt(path) : 16);
|
|
2800
4237
|
if (options.copy) pbcopy(password);
|
|
2801
4238
|
else console.log(password);
|
|
2802
4239
|
},
|
|
4240
|
+
/**
|
|
4241
|
+
* @method sops-setup
|
|
4242
|
+
* @description End-to-end SOPS/Age onboarding for a host: installs tooling, generates the Age
|
|
4243
|
+
* keypair and creation rules, pins the key path for non-interactive runs, encrypts the
|
|
4244
|
+
* requested Secrets into the Git-tracked store, then validates and applies them.
|
|
4245
|
+
*
|
|
4246
|
+
* Every step is idempotent and re-runnable. Notably it delegates key generation to
|
|
4247
|
+
* `secret sops --init` rather than calling `age-keygen` directly: a bare `age-keygen -o`
|
|
4248
|
+
* overwrites an existing key, which would orphan every manifest already encrypted to the
|
|
4249
|
+
* previous recipient with no way to recover them.
|
|
4250
|
+
*
|
|
4251
|
+
* On a host that pulled a store created elsewhere, the freshly generated key is not a recipient
|
|
4252
|
+
* of the inherited manifests. `init()` registers this host in the creation rules so what it
|
|
4253
|
+
* encrypts from here on stays readable, but existing manifests can only be re-keyed from a host
|
|
4254
|
+
* that still holds a decrypting key. That case is reported per secret and then raised by the
|
|
4255
|
+
* apply pre-flight with the available remedies, rather than surfacing as a sops decrypt error.
|
|
4256
|
+
*
|
|
4257
|
+
* Onboards the whole self-hosted data tier by default — PostgreSQL, MariaDB, and MongoDB
|
|
4258
|
+
* (`postgres-secret`, `mariadb-secret`, `mongodb-secret`, `mongodb-keyfile`). The MongoDB
|
|
4259
|
+
* keyfile is included because the StatefulSet mounts it for intra-replica-set auth and will
|
|
4260
|
+
* not start without it. Pass an explicit comma-separated list to narrow the set.
|
|
4261
|
+
*
|
|
4262
|
+
* Secret values are resolved per data key, in order:
|
|
4263
|
+
* 1. the origin seed file, when one exists (`engine-private/postgresql-password`) — this is
|
|
4264
|
+
* the real onboarding path, carrying the credential the cluster already runs on;
|
|
4265
|
+
* 2. `--args` as `key=value` pairs, for a value supplied by the operator;
|
|
4266
|
+
* 3. a freshly generated value: a base64 keyfile for `mongodb-keyfile`, `admin` for a
|
|
4267
|
+
* `username`, otherwise a 24-character secure password.
|
|
4268
|
+
*
|
|
4269
|
+
* Plaintext manifests are written by Node under `/dev/shm` at mode 600 and shredded by
|
|
4270
|
+
* `encrypt()`. They are never emitted through a shell heredoc, which would place the
|
|
4271
|
+
* credential in the command string and therefore in the process table and the command log.
|
|
4272
|
+
*
|
|
4273
|
+
* Usage:
|
|
4274
|
+
* underpost run sops-setup # postgres + mariadb + mongo
|
|
4275
|
+
* underpost run sops-setup mongodb-secret,mongodb-keyfile --namespace prod
|
|
4276
|
+
* underpost run sops-setup postgres-secret --args "password=s3cr3t"
|
|
4277
|
+
* underpost run sops-setup --dry-run # stop before mutating cluster
|
|
4278
|
+
* underpost run sops-setup --force # replace stored manifests
|
|
4279
|
+
* @param {string} path - Comma-separated Secret names to onboard. Defaults to the full data
|
|
4280
|
+
* tier: postgres-secret, mariadb-secret, mongodb-secret, mongodb-keyfile.
|
|
4281
|
+
* @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
|
|
4282
|
+
* @param {string} options.namespace - Target namespace for the store and the apply (default: 'default').
|
|
4283
|
+
* @param {string} options.args - Comma-separated `key=value` overrides for Secret data keys.
|
|
4284
|
+
* @param {boolean} options.dryRun - Validate and server-dry-run only; never apply.
|
|
4285
|
+
* @param {boolean} options.force - Replace encrypted manifests that already exist.
|
|
4286
|
+
* @memberof UnderpostRun
|
|
4287
|
+
*/
|
|
4288
|
+
'sops-setup': (path = '', options = DEFAULT_OPTION) => {
|
|
4289
|
+
const namespace = options.namespace || 'default';
|
|
4290
|
+
const secretNames = (path || SOPS_SETUP_DEFAULT_SECRETS.join(','))
|
|
4291
|
+
.split(',')
|
|
4292
|
+
.map((name) => name.trim())
|
|
4293
|
+
.filter(Boolean);
|
|
4294
|
+
|
|
4295
|
+
// `--args key=value,key2=value2` overrides, applied to any secret that declares that key.
|
|
4296
|
+
const overrides = `${options.args || ''}`.split(',').reduce((acc, pair) => {
|
|
4297
|
+
const separator = pair.indexOf('=');
|
|
4298
|
+
if (separator > 0) acc[pair.slice(0, separator).trim()] = pair.slice(separator + 1).trim();
|
|
4299
|
+
return acc;
|
|
4300
|
+
}, {});
|
|
4301
|
+
|
|
4302
|
+
logger.info('sops-setup', { secretNames, namespace, dryRun: !!options.dryRun, force: !!options.force });
|
|
4303
|
+
|
|
4304
|
+
// 1. Host tooling, then keypair + creation rules. Both no-op when already present.
|
|
4305
|
+
Underpost.secret.sops.installTooling();
|
|
4306
|
+
Underpost.secret.sops.init();
|
|
4307
|
+
|
|
4308
|
+
// 2. Pin the resolved key path for non-interactive runs (systemd units, CronJobs, sudo).
|
|
4309
|
+
// Written with the concrete path rather than a guessed default, because `sudo` resets
|
|
4310
|
+
// HOME and a wrong guess surfaces later as an opaque decrypt failure.
|
|
4311
|
+
const keyFile = Underpost.secret.sops.keyFile();
|
|
4312
|
+
shellExec(
|
|
4313
|
+
`sudo tee /etc/profile.d/underpost-sops.sh >/dev/null <<'UNDERPOST_SOPS_ENV_EOF'
|
|
4314
|
+
export SOPS_AGE_KEY_FILE="\${SOPS_AGE_KEY_FILE:-${keyFile}}"
|
|
4315
|
+
UNDERPOST_SOPS_ENV_EOF`,
|
|
4316
|
+
);
|
|
4317
|
+
shellExec(`sudo chmod 644 /etc/profile.d/underpost-sops.sh`);
|
|
4318
|
+
|
|
4319
|
+
// 3. Build and encrypt each requested Secret.
|
|
4320
|
+
const stageDir = '/dev/shm/underpost-secrets';
|
|
4321
|
+
const held = Underpost.secret.sops.localRecipients();
|
|
4322
|
+
fs.ensureDirSync(stageDir);
|
|
4323
|
+
fs.chmodSync(stageDir, 0o700);
|
|
4324
|
+
try {
|
|
4325
|
+
for (const name of secretNames) {
|
|
4326
|
+
const stored = Underpost.secret.sops.has(name, namespace);
|
|
4327
|
+
if (stored && !options.force) {
|
|
4328
|
+
// A stored manifest this host cannot open is present but unusable here, so reporting it
|
|
4329
|
+
// as onboarded would send the operator on to an apply that is guaranteed to fail.
|
|
4330
|
+
if (Underpost.secret.sops.decryptable(Underpost.secret.sops.manifestPath(name, namespace), held))
|
|
4331
|
+
logger.info(`${name} is already onboarded in ns/${namespace}; skipping (use --force to replace)`);
|
|
4332
|
+
else
|
|
4333
|
+
logger.warn(
|
|
4334
|
+
`${name} is stored in ns/${namespace} but is sealed to an Age recipient this host does not hold; ` +
|
|
4335
|
+
`skipping. Adopt the store's key, re-key it from a host that holds one, or re-onboard from the ` +
|
|
4336
|
+
`origin seed files with --force.`,
|
|
4337
|
+
);
|
|
4338
|
+
continue;
|
|
4339
|
+
}
|
|
4340
|
+
|
|
4341
|
+
// Data keys come from the secret's origin seed contract, so an onboarded manifest
|
|
4342
|
+
// carries exactly the keys the workload's secretKeyRef already expects.
|
|
4343
|
+
const seedSources = Underpost.secret.sops.seedSources(name);
|
|
4344
|
+
const dataKeys = Object.keys(seedSources).length > 0 ? Object.keys(seedSources) : ['password'];
|
|
4345
|
+
const stringData = {};
|
|
4346
|
+
for (const key of dataKeys) {
|
|
4347
|
+
const seedPath = seedSources[key];
|
|
4348
|
+
if (seedPath && fs.existsSync(seedPath)) {
|
|
4349
|
+
stringData[key] = fs.readFileSync(seedPath, 'utf8').trim();
|
|
4350
|
+
logger.info(`${name}.${key} seeded from ${seedPath}`);
|
|
4351
|
+
} else if (overrides[key] !== undefined) {
|
|
4352
|
+
stringData[key] = overrides[key];
|
|
4353
|
+
logger.info(`${name}.${key} taken from --args`);
|
|
4354
|
+
} else {
|
|
4355
|
+
stringData[key] = generateSeedValue(key);
|
|
4356
|
+
// Replacing a stored manifest with a value nothing seeded means the credential the
|
|
4357
|
+
// running datastore still authenticates against is being thrown away.
|
|
4358
|
+
if (stored)
|
|
4359
|
+
logger.warn(
|
|
4360
|
+
`${name}.${key} generated while replacing the stored manifest — no seed file at ` +
|
|
4361
|
+
`${seedPath || '(unmapped)'} and no --args override. The running datastore keeps its old ` +
|
|
4362
|
+
`credential until this value is applied to it; pass --args "${key}=<value>" to keep the ` +
|
|
4363
|
+
`existing one.`,
|
|
4364
|
+
);
|
|
4365
|
+
else logger.info(`${name}.${key} generated`);
|
|
4366
|
+
}
|
|
4367
|
+
}
|
|
4368
|
+
|
|
4369
|
+
const stagePath = `${stageDir}/${name}.yaml`;
|
|
4370
|
+
fs.outputFileSync(
|
|
4371
|
+
stagePath,
|
|
4372
|
+
[
|
|
4373
|
+
'apiVersion: v1',
|
|
4374
|
+
'kind: Secret',
|
|
4375
|
+
'metadata:',
|
|
4376
|
+
` name: ${name}`,
|
|
4377
|
+
` namespace: ${namespace}`,
|
|
4378
|
+
' labels:',
|
|
4379
|
+
' app.kubernetes.io/managed-by: underpost',
|
|
4380
|
+
'type: Opaque',
|
|
4381
|
+
'stringData:',
|
|
4382
|
+
// Single-quoted YAML scalars with doubled internal quotes: values are generated or
|
|
4383
|
+
// operator-supplied and may contain characters YAML would otherwise interpret.
|
|
4384
|
+
...Object.entries(stringData).map(([key, value]) => ` ${key}: '${`${value}`.replace(/'/g, "''")}'`),
|
|
4385
|
+
'',
|
|
4386
|
+
].join('\n'),
|
|
4387
|
+
'utf8',
|
|
4388
|
+
);
|
|
4389
|
+
fs.chmodSync(stagePath, 0o600);
|
|
4390
|
+
// encrypt() stages, validates, moves into place, and shreds the plaintext source.
|
|
4391
|
+
Underpost.secret.sops.encrypt(stagePath, namespace, options);
|
|
4392
|
+
}
|
|
4393
|
+
} finally {
|
|
4394
|
+
// Defense in depth: encrypt() shreds each source, but a throw mid-loop must not leave a
|
|
4395
|
+
// plaintext manifest sitting in shared memory.
|
|
4396
|
+
fs.removeSync(stageDir);
|
|
4397
|
+
}
|
|
4398
|
+
|
|
4399
|
+
Underpost.secret.sops.list();
|
|
4400
|
+
|
|
4401
|
+
// 4. Validate every manifest in the namespace, then apply unless this is a dry run.
|
|
4402
|
+
Underpost.secret.sops.apply(namespace, { dryRun: true });
|
|
4403
|
+
if (options.dryRun) return logger.info('--dry-run: validated only, cluster left unchanged');
|
|
4404
|
+
Underpost.secret.sops.apply(namespace);
|
|
4405
|
+
},
|
|
4406
|
+
/**
|
|
4407
|
+
* @method sops-status
|
|
4408
|
+
* @description Reports the live state of the SOPS/Age secret system: host tooling, the Age
|
|
4409
|
+
* key and its recipient, the committed creation rules, every stored manifest with whether the
|
|
4410
|
+
* local key can open it and whether the cluster still matches, and which managed Secrets are
|
|
4411
|
+
* onboarded versus still seeding from their origin path.
|
|
4412
|
+
*
|
|
4413
|
+
* Read-only and safe to run anywhere. Decryption happens only for the drift check, only for
|
|
4414
|
+
* manifests the local key is a recipient of, and only into `kubectl diff` with its output
|
|
4415
|
+
* discarded — no secret value is ever printed or written to disk.
|
|
4416
|
+
*
|
|
4417
|
+
* Usage:
|
|
4418
|
+
* underpost run sops-status # every managed key, ns default
|
|
4419
|
+
* underpost run sops-status mongo # partial match: both mongo keys
|
|
4420
|
+
* underpost run sops-status --namespace prod # every managed key in ns prod
|
|
4421
|
+
* @param {string} path - Comma-separated managed Secret keys to report on; empty reports all.
|
|
4422
|
+
* Matched as case-insensitive substrings (`mongo` selects mongodb-secret and mongodb-keyfile).
|
|
4423
|
+
* Filters both the stored-manifest listing and the coverage table.
|
|
4424
|
+
* @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
|
|
4425
|
+
* @param {string} options.namespace - Namespace to inspect (DEFAULT_OPTION scheme, default 'default').
|
|
4426
|
+
* @memberof UnderpostRun
|
|
4427
|
+
*/
|
|
4428
|
+
'sops-status': (path = '', options = DEFAULT_OPTION) => {
|
|
4429
|
+
const sops = Underpost.secret.sops;
|
|
4430
|
+
// `--namespace` selects the namespace (DEFAULT_OPTION scheme); `path` narrows which managed
|
|
4431
|
+
// Secret keys to report on, so the two axes stay independent.
|
|
4432
|
+
const namespace = options.namespace || 'default';
|
|
4433
|
+
const manageSecretKeyFilter = path
|
|
4434
|
+
.split(',')
|
|
4435
|
+
.map((key) => key.trim().toLowerCase())
|
|
4436
|
+
.filter(Boolean);
|
|
4437
|
+
// Partial, case-insensitive substring match, so `mongo` reaches both `mongodb-secret` and
|
|
4438
|
+
// `mongodb-keyfile` without having to spell either out.
|
|
4439
|
+
const matchesKeyFilter = (name) =>
|
|
4440
|
+
manageSecretKeyFilter.length === 0 || manageSecretKeyFilter.some((key) => name.toLowerCase().includes(key));
|
|
4441
|
+
const mark = (ok) => (ok ? 'yes' : 'no');
|
|
4442
|
+
|
|
4443
|
+
// ── Tooling ────────────────────────────────────────────────────────────
|
|
4444
|
+
const version = (bin, flag) =>
|
|
4445
|
+
sops.hasBinary(bin)
|
|
4446
|
+
? shellExec(`${bin} ${flag} 2>/dev/null | head -1`, { stdout: true, silent: true, disableLog: true }).trim()
|
|
4447
|
+
: '(not installed)';
|
|
4448
|
+
logger.info(
|
|
4449
|
+
'[sops-status] Tooling\n' +
|
|
4450
|
+
` sops ${version('sops', '--version')}\n` +
|
|
4451
|
+
` age ${version('age', '--version')}\n` +
|
|
4452
|
+
` age-keygen ${sops.hasBinary('age-keygen') ? 'installed' : '(not installed)'}`,
|
|
4453
|
+
);
|
|
4454
|
+
|
|
4455
|
+
// ── Age key ────────────────────────────────────────────────────────────
|
|
4456
|
+
const keyFile = sops.keyFile();
|
|
4457
|
+
const keyExists = fs.existsSync(keyFile);
|
|
4458
|
+
// A key file may hold several identities — that is how a host joins a store it did not
|
|
4459
|
+
// create — so every check below works against the whole held set, not one recipient.
|
|
4460
|
+
const held = sops.localRecipients();
|
|
4461
|
+
const keyMode = keyExists ? (fs.statSync(keyFile).mode & 0o777).toString(8) : '';
|
|
4462
|
+
logger.info(
|
|
4463
|
+
'[sops-status] Age key\n' +
|
|
4464
|
+
` path ${keyFile}\n` +
|
|
4465
|
+
` present ${mark(keyExists)}${keyExists ? ` (mode ${keyMode}${keyMode === '600' || keyMode === '400' ? '' : ' — INSECURE, run chmod 600'})` : ''}\n` +
|
|
4466
|
+
` recipients ${held.join(', ') || (keyExists ? '(none — unreadable key file)' : '(none)')}` +
|
|
4467
|
+
(keyExists ? '' : `\n searched ${sops.keyFileCandidates().join(', ')}`),
|
|
4468
|
+
);
|
|
4469
|
+
|
|
4470
|
+
// ── Creation rules ─────────────────────────────────────────────────────
|
|
4471
|
+
const confPath = './engine-private/secrets/.sops.yaml';
|
|
4472
|
+
const ruleRecipients = sops.creationRecipients();
|
|
4473
|
+
logger.info(
|
|
4474
|
+
'[sops-status] Creation rules\n' +
|
|
4475
|
+
` config ${confPath} ${fs.existsSync(confPath) ? '' : '(missing — run: underpost secret sops --init)'}\n` +
|
|
4476
|
+
` recipients ${ruleRecipients.length > 0 ? ruleRecipients.join(', ') : '(none)'}\n` +
|
|
4477
|
+
` local key listed ${mark(held.some((recipient) => ruleRecipients.includes(recipient)))}`,
|
|
4478
|
+
);
|
|
4479
|
+
|
|
4480
|
+
// ── Stored manifests ───────────────────────────────────────────────────
|
|
4481
|
+
const manifests = sops.manifests(namespace).filter((manifest) => matchesKeyFilter(manifest.name));
|
|
4482
|
+
const onboarded = new Set();
|
|
4483
|
+
if (manifests.length === 0)
|
|
4484
|
+
logger.warn(
|
|
4485
|
+
`[sops-status] Store\n no encrypted manifests in ns/${namespace}` +
|
|
4486
|
+
(manageSecretKeyFilter.length > 0 ? ` matching ${manageSecretKeyFilter.join(', ')}` : ''),
|
|
4487
|
+
);
|
|
4488
|
+
else {
|
|
4489
|
+
const rows = manifests.map((manifest) => {
|
|
4490
|
+
onboarded.add(manifest.name);
|
|
4491
|
+
const recipients = sops.manifestRecipients(manifest.path);
|
|
4492
|
+
const decryptable = sops.decryptable(manifest.path, held);
|
|
4493
|
+
const live = shellExec(
|
|
4494
|
+
`kubectl get secret ${manifest.name} -n ${manifest.namespace} --ignore-not-found -o name 2>/dev/null || true`,
|
|
4495
|
+
{ stdout: true, silent: true, silentOnError: true, disableLog: true },
|
|
4496
|
+
).trim();
|
|
4497
|
+
// Drift is decided by kubectl's exit code; its stdout would contain the decrypted
|
|
4498
|
+
// values, so it is discarded rather than captured.
|
|
4499
|
+
let sync = 'n/a';
|
|
4500
|
+
if (live && decryptable) {
|
|
4501
|
+
const result = shellExec(
|
|
4502
|
+
`bash -c 'set -o pipefail; SOPS_AGE_KEY_FILE="${keyFile}" sops --decrypt "${manifest.path}" ` +
|
|
4503
|
+
`| kubectl diff -f - -n "${manifest.namespace}" >/dev/null 2>&1'`,
|
|
4504
|
+
{ silentOnError: true, disableLog: true, stdout: false },
|
|
4505
|
+
);
|
|
4506
|
+
sync = result.code === 0 ? 'in-sync' : result.code === 1 ? 'DRIFT' : 'error';
|
|
4507
|
+
} else if (!live) sync = 'not applied';
|
|
4508
|
+
else if (!decryptable) sync = 'no local key';
|
|
4509
|
+
return (
|
|
4510
|
+
` ${`${manifest.namespace}/${manifest.name}`.padEnd(34)} ` +
|
|
4511
|
+
`recipients=${String(recipients.length).padEnd(3)} ` +
|
|
4512
|
+
`decryptable=${mark(decryptable).padEnd(4)} ` +
|
|
4513
|
+
`live=${mark(!!live).padEnd(4)} ` +
|
|
4514
|
+
`${sync}`
|
|
4515
|
+
);
|
|
4516
|
+
});
|
|
4517
|
+
logger.info(`[sops-status] Store — ns/${namespace} (${manifests.length} manifest(s))\n` + rows.join('\n'));
|
|
4518
|
+
}
|
|
4519
|
+
|
|
4520
|
+
// ── Coverage ───────────────────────────────────────────────────────────
|
|
4521
|
+
const coverage = sops
|
|
4522
|
+
.managedSecrets()
|
|
4523
|
+
.filter(matchesKeyFilter)
|
|
4524
|
+
.map((name) => {
|
|
4525
|
+
const seeds = Object.values(sops.seedSources(name));
|
|
4526
|
+
const seedPresent = seeds.length > 0 && seeds.every((seed) => fs.existsSync(seed));
|
|
4527
|
+
const source = onboarded.has(name)
|
|
4528
|
+
? 'sops'
|
|
4529
|
+
: seedPresent
|
|
4530
|
+
? 'origin seed'
|
|
4531
|
+
: seeds.length
|
|
4532
|
+
? 'MISSING'
|
|
4533
|
+
: 'unmapped';
|
|
4534
|
+
return ` ${name.padEnd(24)} ${source.padEnd(12)} ${seeds.length ? `seed=${mark(seedPresent)}` : ''}`;
|
|
4535
|
+
});
|
|
4536
|
+
if (coverage.length === 0)
|
|
4537
|
+
logger.warn(
|
|
4538
|
+
`[sops-status] Coverage\n no managed Secret matches ${manageSecretKeyFilter.join(', ')}\n` +
|
|
4539
|
+
` known keys: ${sops.managedSecrets().join(', ')}`,
|
|
4540
|
+
);
|
|
4541
|
+
else
|
|
4542
|
+
logger.info('[sops-status] Coverage (which source each managed Secret deploys from)\n' + coverage.join('\n'));
|
|
4543
|
+
},
|
|
2803
4544
|
/**
|
|
2804
4545
|
* @method secret
|
|
2805
4546
|
* @description Creates an Underpost secret named 'underpost' from a file, defaulting to `/home/dd/engine/engine-private/conf/dd-cron/.env.production` if no path is provided.
|
|
@@ -2920,7 +4661,7 @@ EOF`);
|
|
|
2920
4661
|
|
|
2921
4662
|
const envs = Underpost.env.list();
|
|
2922
4663
|
|
|
2923
|
-
const cmd = `kubectl apply -f - <<EOF
|
|
4664
|
+
const cmd = `kubectl apply -f - <<'EOF'
|
|
2924
4665
|
apiVersion: ${apiVersion}
|
|
2925
4666
|
kind: ${kindType}
|
|
2926
4667
|
metadata:
|