underpost 3.2.80 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ghpkg.ci.yml +7 -1
- package/.github/workflows/pwa-microservices-template-page.cd.yml +1 -16
- package/.github/workflows/pwa-microservices-template-test.ci.yml +1 -1
- package/.github/workflows/release.cd.yml +1 -9
- package/CHANGELOG.md +291 -1
- package/CLI-HELP.md +174 -23
- package/README.md +5 -2
- package/bin/build.js +7 -5
- package/bin/deploy.js +19 -17
- package/deploy/lib/logging.sh +96 -0
- package/deploy/pwa-microservices-template/deploy.sh +72 -0
- package/deploy/release/deploy.sh +62 -0
- package/docker-compose.yml +1 -1
- package/manifests/cronjobs/dd-cron/dd-cron-backup.yaml +5 -1
- package/manifests/cronjobs/dd-cron/dd-cron-dns.yaml +1 -1
- package/manifests/cronjobs/dd-cron/dd-cron-vultr.yaml +52 -0
- package/manifests/deployment/dd-default-development/deployment.yaml +2 -2
- package/manifests/deployment/playwright/deployment.yaml +1 -1
- package/manifests/mongodb/kustomization.yaml +4 -1
- package/manifests/mongodb/statefulset.yaml +4 -0
- package/manifests/mongodb/storage-class.yaml +9 -2
- package/package.json +19 -19
- package/scripts/audit-selinux.sh +64 -0
- package/scripts/coverall-test.sh +24 -0
- package/scripts/gpu-diag.sh +0 -0
- package/scripts/ip-info.sh +0 -0
- package/scripts/k3s-node-setup.sh +18 -15
- package/scripts/kubeadm-node-setup.sh +12 -23
- package/scripts/link-local-underpost-cli.sh +0 -0
- package/scripts/lxd-vm-setup.sh +0 -0
- package/scripts/maas-nat-firewalld.sh +0 -0
- package/scripts/nat-iptables.sh +12 -4
- package/scripts/rhel-grpc-setup.sh +0 -0
- package/scripts/rocky-kickstart.sh +25 -9
- package/scripts/test-monitor.sh +4 -3
- package/src/cli/baremetal.js +1 -2
- package/src/cli/cloud-init.js +1 -1
- package/src/cli/cluster.js +786 -96
- package/src/cli/db.js +11 -4
- package/src/cli/deploy.js +1698 -177
- package/src/cli/docker-compose.js +19 -178
- package/src/cli/env.js +1 -1
- package/src/cli/image.js +15 -7
- package/src/cli/index.js +245 -44
- package/src/cli/ipfs.js +82 -11
- package/src/cli/lxd.js +1 -1
- package/src/cli/monitor.js +2 -2
- package/src/cli/release.js +57 -22
- package/src/cli/repository.js +12 -10
- package/src/cli/run.js +2195 -427
- package/src/cli/secrets.js +969 -0
- package/src/cli/ssh.js +206 -105
- package/src/cli/system.js +26 -13
- package/src/cli/test.js +1 -1
- package/src/cli/vultr.js +583 -0
- package/src/cli/wireguard.js +2125 -0
- package/src/client-builder/client-build.js +102 -13
- package/src/client-builder/ssr.js +27 -73
- package/src/db/mongo/MongoBootstrap.js +295 -54
- package/src/db/mongo/MongooseDB.js +51 -32
- package/src/index.js +25 -1
- package/src/projects/underpost/catalog-underpost.js +4 -1
- package/src/server/backup.js +1 -1
- package/src/server/conf.js +1216 -168
- package/src/server/cri.js +70 -0
- package/src/server/cron.js +249 -51
- package/src/server/dns.js +100 -6
- package/src/server/environment.js +98 -0
- package/src/server/forward-proxy.js +549 -0
- package/src/server/middlewares.js +56 -1
- package/src/server/process.js +0 -1
- package/src/server/selinux.js +185 -0
- package/src/server/systemd.js +205 -0
- package/src/server/underpost-compression.js +186 -0
- package/src/server/underpost-gateway.js +1083 -0
- package/src/server/underpost-ingress.js +380 -0
- package/test/cluster-instances.test.js +435 -0
- package/test/deploy-node-placement.test.js +45 -0
- package/test/instance-traffic-plan.test.js +710 -0
- package/test/selinux.test.js +71 -0
- package/test/sops-secret-store.test.js +612 -0
- package/test/underpost-gateway.test.js +510 -0
- package/test/underpost-ingress.test.js +305 -0
- package/test/wireguard-edge.test.js +1177 -0
package/src/cli/cluster.js
CHANGED
|
@@ -4,9 +4,26 @@
|
|
|
4
4
|
* @namespace UnderpostCluster
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
-
import {
|
|
7
|
+
import { clusterTypeFactory, gatewayApiEnabledFactory, resolveReplicaCount } from '../server/conf.js';
|
|
8
|
+
import { getNpmRootPath, HOST_VOLUME_ROOT } from '../server/environment.js';
|
|
8
9
|
import { loggerFactory } from '../server/logger.js';
|
|
9
10
|
import { shellExec } from '../server/process.js';
|
|
11
|
+
import { crictlCommandFactory, resolveCriSocket } from '../server/cri.js';
|
|
12
|
+
import {
|
|
13
|
+
runSELinuxCommands,
|
|
14
|
+
selinuxContainerSharedContextCommandsFactory,
|
|
15
|
+
selinuxEnforcingCommandsFactory,
|
|
16
|
+
selinuxPackagesCommandFactory,
|
|
17
|
+
selinuxRestoreconCommandFactory,
|
|
18
|
+
} from '../server/selinux.js';
|
|
19
|
+
import { UNDERPOST_GATEWAY, seedDefaultStatusPage } from '../server/underpost-gateway.js';
|
|
20
|
+
import {
|
|
21
|
+
UNDERPOST_INGRESS,
|
|
22
|
+
gatewayBackendFactory,
|
|
23
|
+
underpostIngressConfFactory,
|
|
24
|
+
underpostIngressHostMapFactory,
|
|
25
|
+
underpostIngressManifestsFactory,
|
|
26
|
+
} from '../server/underpost-ingress.js';
|
|
10
27
|
import { MONGODB_DEFAULT_REPLICA_COUNT } from '../db/mongo/MongooseDB.js';
|
|
11
28
|
import { MongoBootstrap } from '../db/mongo/MongoBootstrap.js';
|
|
12
29
|
import os from 'os';
|
|
@@ -15,6 +32,32 @@ import Underpost from '../index.js';
|
|
|
15
32
|
|
|
16
33
|
const logger = loggerFactory(import.meta);
|
|
17
34
|
|
|
35
|
+
const GATEWAY_API_RELEASE = 'v1.5.1';
|
|
36
|
+
const ENVOY_GATEWAY_VERSION = 'v1.8.3';
|
|
37
|
+
const CONTOUR_NAMESPACE = 'projectcontour';
|
|
38
|
+
const KUBEADM_CONTAINER_MOUNT_PATHS = ['/etc/kubernetes', '/var/lib/etcd', '/var/lib/calico'];
|
|
39
|
+
const HOST_VOLUME_PATHS = ['/data', '/opt/local-path-provisioner', HOST_VOLUME_ROOT];
|
|
40
|
+
const K3S_SELINUX_PATHS = [
|
|
41
|
+
'/usr/local/bin/k3s',
|
|
42
|
+
'/etc/rancher',
|
|
43
|
+
'/var/lib/rancher',
|
|
44
|
+
'/var/lib/kubelet',
|
|
45
|
+
'/var/lib/cni',
|
|
46
|
+
];
|
|
47
|
+
|
|
48
|
+
const shareWithContainers = (paths = []) => {
|
|
49
|
+
for (const path of paths) shellExec(`sudo mkdir -p ${path}`);
|
|
50
|
+
runSELinuxCommands(selinuxContainerSharedContextCommandsFactory(paths), { execute: shellExec });
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
const enforceSELinux = (paths = []) => {
|
|
54
|
+
shellExec(
|
|
55
|
+
`if [ -f /etc/redhat-release ] && command -v dnf >/dev/null 2>&1 && { ! command -v restorecon >/dev/null 2>&1 || ! command -v semanage >/dev/null 2>&1; }; then ${selinuxPackagesCommandFactory()}; fi`,
|
|
56
|
+
);
|
|
57
|
+
const commands = selinuxEnforcingCommandsFactory({ restorePaths: paths });
|
|
58
|
+
runSELinuxCommands(commands, { execute: shellExec });
|
|
59
|
+
};
|
|
60
|
+
|
|
18
61
|
/**
|
|
19
62
|
* @class UnderpostCluster
|
|
20
63
|
* @description Manages Kubernetes cluster initialization, configuration, and component deployment.
|
|
@@ -41,9 +84,11 @@ class UnderpostCluster {
|
|
|
41
84
|
* @param {boolean} [options.ipfs=false] - Deploy ipfs-cluster statefulset.
|
|
42
85
|
* @param {boolean} [options.info=false] - Display extensive Kubernetes cluster information.
|
|
43
86
|
* @param {boolean} [options.certManager=false] - Deploy Cert-Manager for certificate management.
|
|
87
|
+
* @param {boolean} [options.gatewayApi=false] - Install the Gateway API control plane (CRDs, Envoy Gateway, and the GatewayClass generated Gateways attach to). Exposure follows the environment: host network in `--dev`, NodePort otherwise.
|
|
88
|
+
* @param {string} [options.gatewayClass=''] - GatewayClass name to provision; must match the one baked into generated manifests.
|
|
44
89
|
* @param {boolean} [options.listPods=false] - List Kubernetes pods.
|
|
45
90
|
* @param {boolean} [options.reset=false] - Perform a comprehensive reset of Kubernetes and container environments.
|
|
46
|
-
* @param {boolean} [options.resetMongodb=false] - Perform a targeted reset of MongoDB components without restarting the entire cluster.
|
|
91
|
+
* @param {boolean} [options.resetMongodb=false] - Perform a targeted reset of MongoDB components without restarting the entire cluster. Combined with `--mongodb` it instead wipes the retained volumes as part of that deploy.
|
|
47
92
|
* @param {boolean} [options.dev=false] - Run in development mode (adjusts paths).
|
|
48
93
|
* @param {string} [options.nsUse=''] - Set the current kubectl namespace (creates namespace if it doesn't exist).
|
|
49
94
|
* @param {string} [options.namespace='default'] - Kubernetes namespace for cluster operations.
|
|
@@ -118,7 +163,7 @@ class UnderpostCluster {
|
|
|
118
163
|
|
|
119
164
|
if (options.config) return options.k3s ? Underpost.cluster.configMinimalK3s() : Underpost.cluster.config();
|
|
120
165
|
|
|
121
|
-
if (options.chown) return Underpost.cluster.chown(options
|
|
166
|
+
if (options.chown) return Underpost.cluster.chown(clusterTypeFactory(options));
|
|
122
167
|
|
|
123
168
|
const npmRoot = getNpmRootPath();
|
|
124
169
|
const underpostRoot = options.dev ? '.' : `${npmRoot}/underpost`;
|
|
@@ -165,9 +210,13 @@ class UnderpostCluster {
|
|
|
165
210
|
});
|
|
166
211
|
}
|
|
167
212
|
|
|
168
|
-
// Targeted MongoDB-only reset (does not restart the whole node)
|
|
169
|
-
|
|
170
|
-
|
|
213
|
+
// Targeted MongoDB-only reset (does not restart the whole node). Combined
|
|
214
|
+
// with --mongodb it is a modifier instead: the deploy below wipes the
|
|
215
|
+
// retained volumes before rolling the StatefulSet out. `--reset` cannot
|
|
216
|
+
// serve that purpose — it short-circuits into the whole-node reset above,
|
|
217
|
+
// which is why `initReplicaSet`'s reset branch was unreachable from the CLI.
|
|
218
|
+
if (options.resetMongodb && !options.mongodb) {
|
|
219
|
+
const clusterType = clusterTypeFactory(options);
|
|
171
220
|
return await MongoBootstrap.reset({
|
|
172
221
|
namespace: options.namespace,
|
|
173
222
|
clusterType,
|
|
@@ -200,8 +249,11 @@ class UnderpostCluster {
|
|
|
200
249
|
// balancer. The platform exposes services explicitly via Project
|
|
201
250
|
// Contour / Envoy and NodePort services (see --node-port); leaving the
|
|
202
251
|
// K3s built-ins enabled would bind the same host ports and conflict.
|
|
203
|
-
shellExec(
|
|
252
|
+
shellExec(
|
|
253
|
+
`curl -sfL https://get.k3s.io | sh -s - $(if command -v selinuxenabled >/dev/null 2>&1 && selinuxenabled; then printf '%s' '--selinux'; fi) --disable=traefik --disable=servicelb`,
|
|
254
|
+
);
|
|
204
255
|
logger.info('K3s installation completed.');
|
|
256
|
+
runSELinuxCommands([selinuxRestoreconCommandFactory(K3S_SELINUX_PATHS)], { execute: shellExec });
|
|
205
257
|
|
|
206
258
|
Underpost.cluster.chown('k3s');
|
|
207
259
|
|
|
@@ -213,22 +265,15 @@ class UnderpostCluster {
|
|
|
213
265
|
} else if (options.kubeadm) {
|
|
214
266
|
Underpost.cluster.config();
|
|
215
267
|
Underpost.cluster.natSetup({ underpostRoot });
|
|
268
|
+
shareWithContainers([...KUBEADM_CONTAINER_MOUNT_PATHS, ...HOST_VOLUME_PATHS]);
|
|
216
269
|
logger.info('Initializing Kubeadm control plane...');
|
|
217
270
|
// Set default values if not provided
|
|
218
271
|
const podNetworkCidr = options.podNetworkCidr || '192.168.0.0/16';
|
|
219
272
|
const controlPlaneEndpoint = options.controlPlaneEndpoint || `${os.hostname()}:6443`;
|
|
220
273
|
|
|
221
|
-
// Initialize kubeadm control plane
|
|
222
|
-
//
|
|
223
|
-
const
|
|
224
|
-
const containerdSocket = 'unix:///run/containerd/containerd.sock';
|
|
225
|
-
const criSocket =
|
|
226
|
-
shellExec(`test -S /var/run/crio/crio.sock && echo crio || echo containerd`, {
|
|
227
|
-
stdout: true,
|
|
228
|
-
silent: true,
|
|
229
|
-
}).trim() === 'crio'
|
|
230
|
-
? crioSocket
|
|
231
|
-
: containerdSocket;
|
|
274
|
+
// Initialize kubeadm control plane against whichever CRI runtime the
|
|
275
|
+
// host actually exposes.
|
|
276
|
+
const criSocket = resolveCriSocket(options);
|
|
232
277
|
shellExec(
|
|
233
278
|
`sudo kubeadm init --pod-network-cidr=${podNetworkCidr} --control-plane-endpoint="${controlPlaneEndpoint}" --cri-socket=${criSocket}`,
|
|
234
279
|
);
|
|
@@ -253,12 +298,19 @@ class UnderpostCluster {
|
|
|
253
298
|
shellExec(
|
|
254
299
|
`kubectl apply -f https://cdn.jsdelivr.net/gh/rancher/local-path-provisioner@master/deploy/local-path-storage.yaml`,
|
|
255
300
|
);
|
|
301
|
+
|
|
302
|
+
// The CNI lands asynchronously, and the node stays NotReady until it
|
|
303
|
+
// does. Block here so the deploy steps that follow (MongoDB and the
|
|
304
|
+
// application workloads) schedule onto a node that can actually run
|
|
305
|
+
// them, instead of racing the Calico rollout.
|
|
306
|
+
logger.info('Waiting for the node to become Ready (Calico rollout)...');
|
|
307
|
+
shellExec(`kubectl wait --for=condition=Ready node --all --timeout=300s`);
|
|
256
308
|
} else {
|
|
257
309
|
Underpost.cluster.config();
|
|
258
310
|
Underpost.cluster.natSetup({ underpostRoot });
|
|
259
311
|
// Kind cluster initialization (default for development)
|
|
260
312
|
logger.info('Initializing Kind cluster...');
|
|
261
|
-
const devReplicaCount =
|
|
313
|
+
const devReplicaCount = resolveReplicaCount(options.replicas, MONGODB_DEFAULT_REPLICA_COUNT);
|
|
262
314
|
shellExec(`sudo mkdir -p /data/mongodb`);
|
|
263
315
|
for (let index = 0; index < devReplicaCount; index++) {
|
|
264
316
|
shellExec(`sudo mkdir -p /data/mongodb/v${index}`);
|
|
@@ -310,7 +362,7 @@ class UnderpostCluster {
|
|
|
310
362
|
.readFileSync(`${underpostRoot}/manifests/grafana/deployment.yaml`, 'utf8')
|
|
311
363
|
.replace('{{GF_SERVER_ROOT_URL}}', options.hosts.split(',')[0])}`;
|
|
312
364
|
console.log(yaml);
|
|
313
|
-
shellExec(`kubectl apply -f - -n ${options.namespace} <<EOF
|
|
365
|
+
shellExec(`kubectl apply -f - -n ${options.namespace} <<'EOF'
|
|
314
366
|
${yaml}
|
|
315
367
|
EOF
|
|
316
368
|
`);
|
|
@@ -329,7 +381,7 @@ EOF
|
|
|
329
381
|
.join(',')}]`,
|
|
330
382
|
)}`;
|
|
331
383
|
console.log(yaml);
|
|
332
|
-
shellExec(`kubectl apply -f - -n ${options.namespace} <<EOF
|
|
384
|
+
shellExec(`kubectl apply -f - -n ${options.namespace} <<'EOF'
|
|
333
385
|
${yaml}
|
|
334
386
|
EOF
|
|
335
387
|
`);
|
|
@@ -361,9 +413,14 @@ EOF
|
|
|
361
413
|
await Underpost.ipfs.deploy(options, underpostRoot);
|
|
362
414
|
}
|
|
363
415
|
if (options.mariadb) {
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
416
|
+
// Secrets before workloads: the StatefulSet's secretKeyRef must resolve at pod admission.
|
|
417
|
+
// Encrypted store first; when no manifest exists the secret is seeded from its origin
|
|
418
|
+
// seed path below. A manifest that exists but fails validation raises rather than
|
|
419
|
+
// silently seeding stale credentials.
|
|
420
|
+
if (!Underpost.secret.sops.applyIfPresent('mariadb-secret', options.namespace))
|
|
421
|
+
shellExec(
|
|
422
|
+
`sudo kubectl create secret generic mariadb-secret --from-file=username=/home/dd/engine/engine-private/mariadb-username --from-file=password=/home/dd/engine/engine-private/mariadb-password --dry-run=client -o yaml | kubectl apply -f - -n ${options.namespace}`,
|
|
423
|
+
);
|
|
367
424
|
shellExec(`kubectl delete statefulset mariadb-statefulset -n ${options.namespace} --ignore-not-found`);
|
|
368
425
|
|
|
369
426
|
if (options.pullImage) Underpost.cluster.pullImage('mariadb:latest', options);
|
|
@@ -371,9 +428,10 @@ EOF
|
|
|
371
428
|
shellExec(`kubectl apply -k ${underpostRoot}/manifests/mariadb -n ${options.namespace}`);
|
|
372
429
|
}
|
|
373
430
|
if (options.mysql) {
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
431
|
+
if (!Underpost.secret.sops.applyIfPresent('mysql-secret', options.namespace))
|
|
432
|
+
shellExec(
|
|
433
|
+
`sudo kubectl create secret generic mysql-secret --from-file=username=/home/dd/engine/engine-private/mysql-username --from-file=password=/home/dd/engine/engine-private/mysql-password --dry-run=client -o yaml | kubectl apply -f - -n ${options.namespace}`,
|
|
434
|
+
);
|
|
377
435
|
shellExec(`sudo mkdir -p /mnt/data`);
|
|
378
436
|
shellExec(`sudo chmod 777 /mnt/data`);
|
|
379
437
|
shellExec(`sudo chown -R $(whoami):$(whoami) /mnt/data`);
|
|
@@ -381,9 +439,10 @@ EOF
|
|
|
381
439
|
}
|
|
382
440
|
if (options.postgresql) {
|
|
383
441
|
if (options.pullImage) Underpost.cluster.pullImage('postgres:latest', options);
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
442
|
+
if (!Underpost.secret.sops.applyIfPresent('postgres-secret', options.namespace))
|
|
443
|
+
shellExec(
|
|
444
|
+
`sudo kubectl create secret generic postgres-secret --from-file=password=/home/dd/engine/engine-private/postgresql-password --dry-run=client -o yaml | kubectl apply -f - -n ${options.namespace}`,
|
|
445
|
+
);
|
|
387
446
|
shellExec(`kubectl apply -k ${underpostRoot}/manifests/postgresql -n ${options.namespace}`);
|
|
388
447
|
}
|
|
389
448
|
if (options.mongodb4) {
|
|
@@ -441,13 +500,13 @@ EOF
|
|
|
441
500
|
});
|
|
442
501
|
}
|
|
443
502
|
} else if (options.mongodb) {
|
|
444
|
-
const clusterType = options
|
|
503
|
+
const clusterType = clusterTypeFactory(options);
|
|
445
504
|
await MongoBootstrap.initReplicaSet({
|
|
446
505
|
namespace: options.namespace,
|
|
447
|
-
replicaCount:
|
|
506
|
+
replicaCount: resolveReplicaCount(options.replicas, MONGODB_DEFAULT_REPLICA_COUNT),
|
|
448
507
|
hostList: serviceHostInput,
|
|
449
508
|
pullImage: options.pullImage,
|
|
450
|
-
reset: options.
|
|
509
|
+
reset: options.resetMongodb === true,
|
|
451
510
|
clusterType,
|
|
452
511
|
underpostRoot,
|
|
453
512
|
});
|
|
@@ -459,20 +518,160 @@ EOF
|
|
|
459
518
|
});
|
|
460
519
|
}
|
|
461
520
|
|
|
521
|
+
// Installing one stack must never break the other. Whichever is already
|
|
522
|
+
// present decides whether this install takes the node's 80/443 for itself
|
|
523
|
+
// or joins the underpost ingress, so it is read before anything is applied.
|
|
524
|
+
const ingressPresence = Underpost.cluster.ingressStackPresence();
|
|
525
|
+
const sharedIngress =
|
|
526
|
+
(options.contour && ingressPresence.gateway) || (options.gatewayApi && ingressPresence.contour);
|
|
527
|
+
|
|
462
528
|
if (options.contour) {
|
|
463
529
|
shellExec(
|
|
464
530
|
`kubectl apply -f https://cdn.jsdelivr.net/gh/projectcontour/contour@release-1.33/examples/render/contour.yaml`,
|
|
465
531
|
);
|
|
532
|
+
// The NodePort patch belongs to Contour's own namespace: its `app=envoy`
|
|
533
|
+
// selector only matches the Envoy DaemonSet there. Applied anywhere else
|
|
534
|
+
// the Service is born with no endpoints, and kube-proxy then answers
|
|
535
|
+
// EVERY connection to its ports with an ICMP port-unreachable REJECT —
|
|
536
|
+
// including 443 on every local address, which silently breaks whatever
|
|
537
|
+
// else serves HTTPS on the node.
|
|
538
|
+
Underpost.cluster.pruneEndpointlessService({ name: 'envoy', namespace: options.namespace });
|
|
539
|
+
// Contour's render always claims hostPort 80/443. Left in place beside a
|
|
540
|
+
// Gateway API data plane it would DNAT every packet away from it, so the
|
|
541
|
+
// claim is released and both are reached through the underpost ingress instead.
|
|
542
|
+
if (sharedIngress)
|
|
543
|
+
Underpost.cluster.releaseHostPortClaim({ name: 'envoy', namespace: CONTOUR_NAMESPACE, ports: [80, 443] });
|
|
466
544
|
if (options.kubeadm) {
|
|
467
545
|
// Envoy service might need NodePort for kubeadm
|
|
468
546
|
shellExec(
|
|
469
|
-
`sudo kubectl apply -f ${underpostRoot}/manifests/envoy-service-nodeport.yaml -n ${
|
|
547
|
+
`sudo kubectl apply -f ${underpostRoot}/manifests/envoy-service-nodeport.yaml -n ${CONTOUR_NAMESPACE}`,
|
|
470
548
|
);
|
|
471
549
|
}
|
|
472
550
|
// K3s has a built-in LoadBalancer (Klipper-lb) that can expose services,
|
|
473
551
|
// so a specific NodePort service might not be needed or can be configured differently.
|
|
474
552
|
}
|
|
475
553
|
|
|
554
|
+
if (options.gatewayApi) {
|
|
555
|
+
// Gateway API stack: the CRDs, the implementation that serves them, and
|
|
556
|
+
// the GatewayClass every generated Gateway attaches to. Envoy Gateway is
|
|
557
|
+
// the implementation because the manifests this repo generates use two of
|
|
558
|
+
// its ClientTrafficPolicy extension, which carries the QUIC/HTTP3
|
|
559
|
+
// listener config. Both versions are pinned so a rebuild provisions the
|
|
560
|
+
// same control plane.
|
|
561
|
+
const env = options.dev ? 'development' : 'production';
|
|
562
|
+
// Envoy Gateway provisions and owns its own data plane Service in
|
|
563
|
+
// envoy-gateway-system. Any hand-managed Service elsewhere that publishes
|
|
564
|
+
// the same ports without endpoints would have kube-proxy reject 80/443
|
|
565
|
+
// out from under it, and any DaemonSet holding hostPort 80/443 would
|
|
566
|
+
// swallow the traffic before the data plane's listener sees it.
|
|
567
|
+
Underpost.cluster.pruneEndpointlessService({ name: 'envoy', namespace: options.namespace });
|
|
568
|
+
// Contour keeps serving beside this stack when it is already installed —
|
|
569
|
+
// it only gives up the node's ports, which the underpost ingress takes over.
|
|
570
|
+
// With no Contour present the claim is removed outright, since a leftover
|
|
571
|
+
// DaemonSet from an uninstalled stack has nothing left to serve.
|
|
572
|
+
if (sharedIngress)
|
|
573
|
+
Underpost.cluster.releaseHostPortClaim({ name: 'envoy', namespace: CONTOUR_NAMESPACE, ports: [80, 443] });
|
|
574
|
+
else Underpost.cluster.pruneHostPortClaim({ name: 'envoy', namespace: CONTOUR_NAMESPACE, ports: [80, 443] });
|
|
575
|
+
shellExec(
|
|
576
|
+
`kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/${GATEWAY_API_RELEASE}/standard-install.yaml`,
|
|
577
|
+
);
|
|
578
|
+
shellExec(
|
|
579
|
+
`helm upgrade --install eg oci://docker.io/envoyproxy/gateway-helm --version ${ENVOY_GATEWAY_VERSION} ` +
|
|
580
|
+
`--namespace envoy-gateway-system --create-namespace`,
|
|
581
|
+
);
|
|
582
|
+
// The GatewayClass is rejected while the webhook is still coming up.
|
|
583
|
+
shellExec(
|
|
584
|
+
`kubectl wait --namespace envoy-gateway-system --for=condition=Available --timeout=5m deployment/envoy-gateway`,
|
|
585
|
+
{ silentOnError: true },
|
|
586
|
+
);
|
|
587
|
+
// Validate against the live CRD schema before creating anything. These
|
|
588
|
+
// are vendor CRDs whose optional fields move between releases, and
|
|
589
|
+
// kubectl applies documents independently: without the pre-flight, a
|
|
590
|
+
// field the installed Envoy Gateway does not know leaves a GatewayClass
|
|
591
|
+
// pointing at an EnvoyProxy that never applied.
|
|
592
|
+
const gatewayClassYaml = Underpost.deploy.gatewayClassYamlFactory({
|
|
593
|
+
env,
|
|
594
|
+
options: { ...options, sharedIngress },
|
|
595
|
+
});
|
|
596
|
+
shellExec(`kubectl apply --dry-run=server -f - <<'EOF'
|
|
597
|
+
${gatewayClassYaml}
|
|
598
|
+
EOF
|
|
599
|
+
`);
|
|
600
|
+
shellExec(`kubectl apply -f - <<'EOF'
|
|
601
|
+
${gatewayClassYaml}
|
|
602
|
+
EOF
|
|
603
|
+
`);
|
|
604
|
+
// The static utility that serves status pages and intercepted contexts.
|
|
605
|
+
// It belongs to the gateway tier, not to any one deploy: a single Nginx
|
|
606
|
+
// holds every host's documents, and routes reach it as an ordinary
|
|
607
|
+
// backend — which is what removes the 4096-byte direct-response ceiling.
|
|
608
|
+
//
|
|
609
|
+
// The heredoc delimiter is quoted, as it must be for every generated
|
|
610
|
+
// manifest: the values are already substituted by the template literal,
|
|
611
|
+
// so anything the shell expands here is content. This one carries
|
|
612
|
+
// `nginx.conf`, whose `$uri` an unquoted delimiter deletes — leaving a
|
|
613
|
+
// `try_files` that matches nothing and answers every host's status page
|
|
614
|
+
// with the shared default.
|
|
615
|
+
const underpostGatewayYaml = Underpost.deploy.underpostGatewayYamlFactory(options);
|
|
616
|
+
shellExec(`kubectl apply -f - -n ${options.namespace} <<'EOF'
|
|
617
|
+
${underpostGatewayYaml}
|
|
618
|
+
EOF
|
|
619
|
+
`);
|
|
620
|
+
const underpostGatewayRoot = Underpost.deploy.underpostGatewayRootFactory(options);
|
|
621
|
+
seedDefaultStatusPage(underpostGatewayRoot);
|
|
622
|
+
logger.info('Gateway static utility applied', {
|
|
623
|
+
name: UNDERPOST_GATEWAY.name,
|
|
624
|
+
root: underpostGatewayRoot,
|
|
625
|
+
});
|
|
626
|
+
|
|
627
|
+
logger.info('Gateway API control plane installed', {
|
|
628
|
+
gatewayApiRelease: GATEWAY_API_RELEASE,
|
|
629
|
+
envoyGateway: ENVOY_GATEWAY_VERSION,
|
|
630
|
+
gatewayClass: Underpost.deploy.gatewayApiConfigFactory(options).gatewayClassName,
|
|
631
|
+
env,
|
|
632
|
+
});
|
|
633
|
+
}
|
|
634
|
+
|
|
635
|
+
// Both stacks are now installed, so the node's 80/443 belong to neither of
|
|
636
|
+
// them. Contour was installed first and Envoy Gateway is arriving, or the
|
|
637
|
+
// reverse — either way this is the point where the underpost ingress has to exist,
|
|
638
|
+
// and the data plane that just came up has to stop binding the host.
|
|
639
|
+
if (sharedIngress) {
|
|
640
|
+
if (options.contour && ingressPresence.gateway) {
|
|
641
|
+
// `--contour` against a cluster that already runs Envoy Gateway: its
|
|
642
|
+
// data plane is still on the host network from a single-stack install,
|
|
643
|
+
// so the EnvoyProxy is re-applied to move it behind the underpost ingress.
|
|
644
|
+
const gatewayClassYaml = Underpost.deploy.gatewayClassYamlFactory({
|
|
645
|
+
env: options.dev ? 'development' : 'production',
|
|
646
|
+
options: { ...options, sharedIngress: true },
|
|
647
|
+
});
|
|
648
|
+
shellExec(`kubectl apply -f - <<'EOF'
|
|
649
|
+
${gatewayClassYaml}
|
|
650
|
+
EOF
|
|
651
|
+
`);
|
|
652
|
+
shellExec(`kubectl rollout status deployment/envoy-gateway -n envoy-gateway-system --timeout=5m`, {
|
|
653
|
+
silentOnError: true,
|
|
654
|
+
});
|
|
655
|
+
}
|
|
656
|
+
// The edge binds the node directly, so it can only come up once both data
|
|
657
|
+
// planes have actually let go — not merely once their templates say so.
|
|
658
|
+
if (!Underpost.cluster.awaitHostPortsFree({ ports: [80, 443] })) {
|
|
659
|
+
logger.error('Skipping the underpost ingress: it would not be able to bind the node ports', {
|
|
660
|
+
hint: 'resolve the holder above, then re-run this command',
|
|
661
|
+
});
|
|
662
|
+
} else {
|
|
663
|
+
Underpost.cluster.installUnderpostIngress({ namespace: options.namespace, options });
|
|
664
|
+
shellExec(
|
|
665
|
+
`kubectl rollout status deployment/${UNDERPOST_INGRESS.name} -n ${options.namespace} --timeout=3m`,
|
|
666
|
+
{ silentOnError: true },
|
|
667
|
+
);
|
|
668
|
+
logger.info('Both ingress stacks are live behind the underpost ingress', {
|
|
669
|
+
ingress: UNDERPOST_INGRESS.name,
|
|
670
|
+
note: 'HTTP/3 is served only by the Gateway API data plane; see the underpost-ingress module',
|
|
671
|
+
});
|
|
672
|
+
}
|
|
673
|
+
}
|
|
674
|
+
|
|
476
675
|
if (options.certManager) {
|
|
477
676
|
if (!Underpost.kubectl.get('cert-manager').find((p) => p.STATUS === 'Running')) {
|
|
478
677
|
shellExec(`helm repo add jetstack https://charts.jetstack.io --force-update`);
|
|
@@ -637,27 +836,540 @@ EOF
|
|
|
637
836
|
`for node in $(kind get nodes); do cat ${tarPath} | docker exec -i $node ctr --namespace=k8s.io images import -; done`,
|
|
638
837
|
);
|
|
639
838
|
shellExec(`rm -f ${tarPath}`);
|
|
640
|
-
} else
|
|
641
|
-
//
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
839
|
+
} else {
|
|
840
|
+
// Kubeadm / K3s: pull directly into the active CRI runtime.
|
|
841
|
+
shellExec(crictlCommandFactory(`pull ${image}`, options));
|
|
842
|
+
}
|
|
843
|
+
},
|
|
844
|
+
|
|
845
|
+
/**
|
|
846
|
+
* @method pruneHostPortClaim
|
|
847
|
+
* @description Removes a DaemonSet that reserves node ports the caller needs.
|
|
848
|
+
* A `hostPort` is not a soft claim: the CNI hostport plugin installs a DNAT
|
|
849
|
+
* that rewrites every packet arriving on that port to the claiming pod,
|
|
850
|
+
* ahead of any process listening on the node. A second ingress stack bound
|
|
851
|
+
* to the same port therefore never sees a single packet, and the symptom is
|
|
852
|
+
* a connection refused by the *other* stack — with nothing in its own logs.
|
|
853
|
+
*
|
|
854
|
+
* Only a DaemonSet that actually claims one of `ports` is removed.
|
|
855
|
+
* @param {string} name - DaemonSet name.
|
|
856
|
+
* @param {string} namespace - Namespace to inspect.
|
|
857
|
+
* @param {Array<number>} ports - Host ports the caller requires.
|
|
858
|
+
* @returns {boolean} True when a DaemonSet was pruned.
|
|
859
|
+
* @memberof UnderpostCluster
|
|
860
|
+
*/
|
|
861
|
+
pruneHostPortClaim({ name, namespace, ports = [80, 443] }) {
|
|
862
|
+
const claimed = shellExec(
|
|
863
|
+
`kubectl get daemonset ${name} -n ${namespace} ` +
|
|
864
|
+
`-o jsonpath='{.spec.template.spec.containers[*].ports[*].hostPort}'`,
|
|
865
|
+
{ stdout: true, silent: true, silentOnError: true },
|
|
866
|
+
);
|
|
867
|
+
const claimedPorts = `${claimed || ''}`
|
|
868
|
+
.split(/\s+/)
|
|
869
|
+
.map((port) => parseInt(port, 10))
|
|
870
|
+
.filter((port) => !isNaN(port));
|
|
871
|
+
const conflicts = claimedPorts.filter((port) => ports.includes(port));
|
|
872
|
+
if (conflicts.length === 0) return false;
|
|
873
|
+
logger.warn(`Pruning DaemonSet ${namespace}/${name}: it reserves host ports needed by this ingress stack`, {
|
|
874
|
+
conflicts,
|
|
875
|
+
reason: 'the CNI hostport DNAT redirects those ports before any node listener receives them',
|
|
876
|
+
});
|
|
877
|
+
shellExec(`kubectl delete daemonset ${name} -n ${namespace} --ignore-not-found`);
|
|
878
|
+
return true;
|
|
879
|
+
},
|
|
880
|
+
|
|
881
|
+
/**
|
|
882
|
+
* @method releaseHostPortClaim
|
|
883
|
+
* @description Strips a DaemonSet's `hostPort` claims without removing the
|
|
884
|
+
* workload, so it keeps serving through its ClusterIP.
|
|
885
|
+
*
|
|
886
|
+
* The non-destructive half of {@link UnderpostCluster.pruneHostPortClaim}.
|
|
887
|
+
* Deleting the DaemonSet is right when one stack replaces the other, and
|
|
888
|
+
* wrong when both are meant to stay: here the data plane is still wanted,
|
|
889
|
+
* it just must not hold the node's ports any more — the underpost ingress does.
|
|
890
|
+
*
|
|
891
|
+
* Applied as a JSON Patch, one `remove` op per claiming port. A
|
|
892
|
+
* strategic-merge patch cannot express this: `ports` is a list merged by
|
|
893
|
+
* `containerPort`, so a patch that simply omits `hostPort` merges into the
|
|
894
|
+
* existing element and leaves the claim exactly where it was — reported as
|
|
895
|
+
* `patched (no change)` while the port stays held. Removing an object field
|
|
896
|
+
* does not shift array indices, so every op in one patch stays valid.
|
|
897
|
+
* @param {string} name - DaemonSet name.
|
|
898
|
+
* @param {string} namespace - Namespace to inspect.
|
|
899
|
+
* @param {Array<number>} [ports] - Host ports to release.
|
|
900
|
+
* @returns {boolean} True when the claim is gone afterwards.
|
|
901
|
+
* @memberof UnderpostCluster
|
|
902
|
+
*/
|
|
903
|
+
releaseHostPortClaim({ name, namespace, ports = [80, 443] }) {
|
|
904
|
+
const readClaims = () => {
|
|
905
|
+
const raw = shellExec(
|
|
906
|
+
`kubectl get daemonset ${name} -n ${namespace} -o jsonpath='{.spec.template.spec.containers}'`,
|
|
907
|
+
{ stdout: true, silent: true, silentOnError: true },
|
|
645
908
|
);
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
909
|
+
try {
|
|
910
|
+
const containers = JSON.parse(`${raw || ''}`);
|
|
911
|
+
return Array.isArray(containers) ? containers : null;
|
|
912
|
+
} catch {
|
|
913
|
+
return null;
|
|
914
|
+
}
|
|
915
|
+
};
|
|
916
|
+
|
|
917
|
+
const containers = readClaims();
|
|
918
|
+
if (!containers) return false;
|
|
919
|
+
const operations = [];
|
|
920
|
+
containers.forEach((container, containerIndex) =>
|
|
921
|
+
(container.ports || []).forEach((port, portIndex) => {
|
|
922
|
+
if (port.hostPort === undefined || !ports.includes(port.hostPort)) return;
|
|
923
|
+
operations.push({
|
|
924
|
+
op: 'remove',
|
|
925
|
+
path: `/spec/template/spec/containers/${containerIndex}/ports/${portIndex}/hostPort`,
|
|
926
|
+
});
|
|
927
|
+
}),
|
|
928
|
+
);
|
|
929
|
+
if (operations.length === 0) return false;
|
|
930
|
+
|
|
931
|
+
logger.warn(`Releasing host ports on DaemonSet ${namespace}/${name}: the underpost ingress owns them now`, {
|
|
932
|
+
ports,
|
|
933
|
+
reason:
|
|
934
|
+
'its ClusterIP keeps the data plane reachable, and the hostPort DNAT would outrank the underpost ingress listener',
|
|
935
|
+
});
|
|
936
|
+
shellExec(`kubectl patch daemonset ${name} -n ${namespace} --type=json -p '${JSON.stringify(operations)}'`);
|
|
937
|
+
|
|
938
|
+
// Verified rather than assumed: this silently did nothing once already,
|
|
939
|
+
// and the failure only surfaced later as an unschedulable edge.
|
|
940
|
+
const remaining = (readClaims() || [])
|
|
941
|
+
.flatMap((container) => container.ports || [])
|
|
942
|
+
.filter((port) => ports.includes(port.hostPort));
|
|
943
|
+
if (remaining.length > 0) {
|
|
944
|
+
logger.error(`Failed to release host ports on DaemonSet ${namespace}/${name}`, {
|
|
945
|
+
stillClaimed: remaining.map((port) => port.hostPort),
|
|
946
|
+
});
|
|
947
|
+
return false;
|
|
948
|
+
}
|
|
949
|
+
shellExec(`kubectl rollout status daemonset/${name} -n ${namespace} --timeout=3m`, { silentOnError: true });
|
|
950
|
+
return true;
|
|
951
|
+
},
|
|
952
|
+
|
|
953
|
+
/**
|
|
954
|
+
* @method awaitHostPortsFree
|
|
955
|
+
* @description Blocks until no known ingress data plane still holds the node's ports.
|
|
956
|
+
*
|
|
957
|
+
* Releasing a claim edits a template; the pod holding the socket goes away
|
|
958
|
+
* only once the rollout completes. Applying the edge before that leaves it
|
|
959
|
+
* either unschedulable or crash-looping on `bind() … Address in use`, and
|
|
960
|
+
* neither failure names the pod that is actually holding the port.
|
|
961
|
+
* @param {Array<number>} [ports] - Host ports that must be free.
|
|
962
|
+
* @param {string} [contourNamespace] - Namespace holding Contour.
|
|
963
|
+
* @param {number} [timeoutMs] - How long to wait.
|
|
964
|
+
* @returns {boolean} True once nothing claims them.
|
|
965
|
+
* @memberof UnderpostCluster
|
|
966
|
+
*/
|
|
967
|
+
awaitHostPortsFree({ ports = [80, 443], contourNamespace = CONTOUR_NAMESPACE, timeoutMs = 3 * 60 * 1000 } = {}) {
|
|
968
|
+
const deadline = Date.now() + timeoutMs;
|
|
969
|
+
const holders = () => {
|
|
970
|
+
const found = [];
|
|
971
|
+
const contourPorts = `${
|
|
972
|
+
shellExec(
|
|
973
|
+
`kubectl get daemonset envoy -n ${contourNamespace} ` +
|
|
974
|
+
`-o jsonpath='{.spec.template.spec.containers[*].ports[*].hostPort}'`,
|
|
975
|
+
{ stdout: true, silent: true, silentOnError: true },
|
|
976
|
+
) || ''
|
|
977
|
+
}`
|
|
978
|
+
.split(/\s+/)
|
|
979
|
+
.map((port) => parseInt(port, 10))
|
|
980
|
+
.filter((port) => ports.includes(port));
|
|
981
|
+
if (contourPorts.length > 0) found.push(`${contourNamespace}/envoy hostPort ${contourPorts.join(',')}`);
|
|
982
|
+
// The Gateway API data plane takes the ports through the host network
|
|
983
|
+
// rather than a hostPort, so its claim is the network mode itself.
|
|
984
|
+
const gatewayHostNetwork = `${
|
|
985
|
+
shellExec(
|
|
986
|
+
`kubectl get deployment -n envoy-gateway-system -l app.kubernetes.io/name=envoy ` +
|
|
987
|
+
`-o jsonpath='{.items[*].spec.template.spec.hostNetwork}'`,
|
|
988
|
+
{ stdout: true, silent: true, silentOnError: true },
|
|
989
|
+
) || ''
|
|
990
|
+
}`.trim();
|
|
991
|
+
if (gatewayHostNetwork.includes('true')) found.push('envoy-gateway-system data plane hostNetwork');
|
|
992
|
+
return found;
|
|
993
|
+
};
|
|
994
|
+
|
|
995
|
+
let current = holders();
|
|
996
|
+
while (current.length > 0 && Date.now() < deadline) {
|
|
997
|
+
logger.info('Waiting for the node ports to be released', { ports, heldBy: current });
|
|
998
|
+
shellExec('sleep 5', { silent: true });
|
|
999
|
+
current = holders();
|
|
1000
|
+
}
|
|
1001
|
+
if (current.length > 0) {
|
|
1002
|
+
logger.error('Node ports are still held; the underpost ingress cannot bind them', {
|
|
1003
|
+
ports,
|
|
1004
|
+
heldBy: current,
|
|
1005
|
+
});
|
|
1006
|
+
return false;
|
|
1007
|
+
}
|
|
1008
|
+
return true;
|
|
1009
|
+
},
|
|
1010
|
+
|
|
1011
|
+
/**
|
|
1012
|
+
* @method ingressStackPresence
|
|
1013
|
+
* @description Which ingress data planes are installed right now.
|
|
1014
|
+
*
|
|
1015
|
+
* Read from the cluster rather than from the flags, because the whole point
|
|
1016
|
+
* is to react to what a previous invocation left behind: `--contour` run
|
|
1017
|
+
* against a cluster that already has Envoy Gateway has to behave differently
|
|
1018
|
+
* from `--contour` on an empty one.
|
|
1019
|
+
* @param {string} [contourNamespace] - Namespace holding Contour.
|
|
1020
|
+
* @returns {{contour: boolean, gateway: boolean}} Presence of each stack.
|
|
1021
|
+
* @memberof UnderpostCluster
|
|
1022
|
+
*/
|
|
1023
|
+
ingressStackPresence(contourNamespace = CONTOUR_NAMESPACE) {
|
|
1024
|
+
const exists = (cmd) =>
|
|
1025
|
+
`${shellExec(cmd, { stdout: true, silent: true, silentOnError: true }) || ''}`.trim().length > 0;
|
|
1026
|
+
return {
|
|
1027
|
+
contour: exists(`kubectl get daemonset envoy -n ${contourNamespace} -o name`),
|
|
1028
|
+
gateway: exists(`kubectl get deployment envoy-gateway -n envoy-gateway-system -o name`),
|
|
1029
|
+
};
|
|
1030
|
+
},
|
|
1031
|
+
|
|
1032
|
+
/**
|
|
1033
|
+
* @method refreshUnderpostIngress
|
|
1034
|
+
* @description Rebuilds the underpost ingress host table after routes change.
|
|
1035
|
+
*
|
|
1036
|
+
* The table maps each hostname to the data plane that has a route object for
|
|
1037
|
+
* it, so it goes stale the moment a host is published — a hostname added
|
|
1038
|
+
* after the last build falls through to the default backend, which is the
|
|
1039
|
+
* *other* stack, and answers 404 for a workload that is running perfectly.
|
|
1040
|
+
* Every path that publishes a route therefore ends here.
|
|
1041
|
+
*
|
|
1042
|
+
* A no-op when the ingress is not installed, which is the single-stack case:
|
|
1043
|
+
* there is no table to keep, because the one data plane owns the ports.
|
|
1044
|
+
* The rendered config is unchanged by a blue/green promotion — the table is
|
|
1045
|
+
* host to stack, not host to colour — so an ordinary deploy re-applies the
|
|
1046
|
+
* same bytes and nothing restarts.
|
|
1047
|
+
* @param {string} [namespace] - Namespace holding the ingress.
|
|
1048
|
+
* @param {object} [options] - Cluster options. `ingressNode` is the only
|
|
1049
|
+
* placement override; application `node` / `nodeName` flags are ignored.
|
|
1050
|
+
* @returns {boolean} True when the table was rebuilt.
|
|
1051
|
+
* @memberof UnderpostCluster
|
|
1052
|
+
*/
|
|
1053
|
+
refreshUnderpostIngress({ namespace = 'default', options = {} } = {}) {
|
|
1054
|
+
const installed = `${
|
|
1055
|
+
shellExec(`kubectl get deployment ${UNDERPOST_INGRESS.name} -n ${namespace} -o name`, {
|
|
1056
|
+
stdout: true,
|
|
1057
|
+
silent: true,
|
|
1058
|
+
silentOnError: true,
|
|
1059
|
+
}) || ''
|
|
1060
|
+
}`.trim();
|
|
1061
|
+
if (!installed) return false;
|
|
1062
|
+
return Underpost.cluster.installUnderpostIngress({ namespace, options });
|
|
1063
|
+
},
|
|
1064
|
+
|
|
1065
|
+
/**
|
|
1066
|
+
* @method installUnderpostIngress
|
|
1067
|
+
* @description Installs or refreshes the underpost ingress in front of both data planes.
|
|
1068
|
+
*
|
|
1069
|
+
* The host table is built from the route objects that actually exist, so a
|
|
1070
|
+
* hostname reaches the stack that describes it no matter which flag was used
|
|
1071
|
+
* last. Rebuilt on every call, which is what keeps it correct after routes
|
|
1072
|
+
* move between stacks.
|
|
1073
|
+
* @param {string} [namespace] - Namespace to deploy the underpost ingress into.
|
|
1074
|
+
* @param {object} [options] - Cluster options. Use `ingressNode` to
|
|
1075
|
+
* explicitly place or recover the public listener.
|
|
1076
|
+
* @returns {boolean} True when the underpost ingress was applied.
|
|
1077
|
+
* @memberof UnderpostCluster
|
|
1078
|
+
*/
|
|
1079
|
+
installUnderpostIngress({ namespace = 'default', options = {} } = {}) {
|
|
1080
|
+
const presence = Underpost.cluster.ingressStackPresence();
|
|
1081
|
+
// Each kind names its hosts in its own field, so each is read with its own
|
|
1082
|
+
// jsonpath rather than one expression that happens to tolerate the other's
|
|
1083
|
+
// shape being absent.
|
|
1084
|
+
const HOST_FIELD = { httpproxy: '.spec.virtualhost.fqdn', httproute: '.spec.hostnames[*]' };
|
|
1085
|
+
const hostsOf = (kind) =>
|
|
1086
|
+
`${
|
|
1087
|
+
shellExec(`kubectl get ${kind} -A -o jsonpath='{range .items[*]}{${HOST_FIELD[kind]}}{" "}{end}'`, {
|
|
652
1088
|
stdout: true,
|
|
653
1089
|
silent: true,
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
1090
|
+
silentOnError: true,
|
|
1091
|
+
}) || ''
|
|
1092
|
+
}`
|
|
1093
|
+
.split(/\s+/)
|
|
1094
|
+
.map((host) => host.trim())
|
|
1095
|
+
.filter(Boolean);
|
|
1096
|
+
|
|
1097
|
+
const backends = {};
|
|
1098
|
+
if (presence.contour) backends.contour = UNDERPOST_INGRESS.backends.contour;
|
|
1099
|
+
if (presence.gateway) {
|
|
1100
|
+
// Envoy Gateway names its own data plane Service, so it is discovered
|
|
1101
|
+
// rather than assumed; without it the underpost ingress would proxy to nothing.
|
|
1102
|
+
const service = `${
|
|
1103
|
+
shellExec(
|
|
1104
|
+
`kubectl get svc -n envoy-gateway-system -l app.kubernetes.io/name=envoy -o jsonpath='{.items[0].metadata.name}'`,
|
|
1105
|
+
{ stdout: true, silent: true, silentOnError: true },
|
|
1106
|
+
) || ''
|
|
1107
|
+
}`.trim();
|
|
1108
|
+
if (service) backends.gateway = gatewayBackendFactory(service);
|
|
1109
|
+
else logger.warn('Envoy Gateway is installed but has provisioned no data plane Service yet');
|
|
1110
|
+
}
|
|
1111
|
+
if (Object.keys(backends).length === 0) {
|
|
1112
|
+
logger.warn('No ingress data plane installed; skipping the underpost ingress');
|
|
1113
|
+
return false;
|
|
658
1114
|
}
|
|
1115
|
+
|
|
1116
|
+
const { entries, conflicts } = underpostIngressHostMapFactory({
|
|
1117
|
+
contourHosts: presence.contour ? hostsOf('httpproxy') : [],
|
|
1118
|
+
gatewayHosts: presence.gateway ? hostsOf('httproute') : [],
|
|
1119
|
+
// During migration both route kinds deliberately coexist. Point the
|
|
1120
|
+
// host at the destination stack before the old object is deleted; a
|
|
1121
|
+
// fixed Gateway preference would create a 404 window when migrating in
|
|
1122
|
+
// the opposite direction (Gateway API -> Contour).
|
|
1123
|
+
preferred: gatewayApiEnabledFactory(options) ? 'gateway' : 'contour',
|
|
1124
|
+
});
|
|
1125
|
+
if (conflicts.length > 0)
|
|
1126
|
+
logger.warn('Hosts described by both stacks; the destination stack wins until the old route is removed', {
|
|
1127
|
+
conflicts,
|
|
1128
|
+
destination: gatewayApiEnabledFactory(options) ? 'gateway' : 'contour',
|
|
1129
|
+
});
|
|
1130
|
+
|
|
1131
|
+
const conf = underpostIngressConfFactory({
|
|
1132
|
+
entries,
|
|
1133
|
+
backends,
|
|
1134
|
+
resolver: Underpost.deploy.clusterDnsFactory(),
|
|
1135
|
+
defaultBackend: backends.gateway ? 'gateway' : 'contour',
|
|
1136
|
+
});
|
|
1137
|
+
const liveNode = `${
|
|
1138
|
+
shellExec(
|
|
1139
|
+
`kubectl get deployment ${UNDERPOST_INGRESS.name} -n ${namespace} ` +
|
|
1140
|
+
`-o jsonpath='{.spec.template.spec.nodeSelector.kubernetes\\.io/hostname}'`,
|
|
1141
|
+
{ stdout: true, silent: true, silentOnError: true },
|
|
1142
|
+
) || ''
|
|
1143
|
+
}`.trim();
|
|
1144
|
+
// `node` and `nodeName` place application workloads. Reusing either here
|
|
1145
|
+
// relocates the public listener during an ordinary deploy; with Recreate
|
|
1146
|
+
// that first deletes the healthy edge and can leave the whole site on
|
|
1147
|
+
// connection-refused if the destination cannot pull Nginx. Only the
|
|
1148
|
+
// dedicated ingressNode option may move this workload. Every route-table
|
|
1149
|
+
// refresh otherwise preserves the established edge node.
|
|
1150
|
+
const requestedNode = options.ingressNode || '';
|
|
1151
|
+
const chosenNode =
|
|
1152
|
+
requestedNode ||
|
|
1153
|
+
liveNode ||
|
|
1154
|
+
Underpost.deploy.resolveDeployNode({
|
|
1155
|
+
node: '',
|
|
1156
|
+
kind: options.kind,
|
|
1157
|
+
kubeadm: options.kubeadm,
|
|
1158
|
+
k3s: options.k3s,
|
|
1159
|
+
env: options.dev ? 'development' : 'production',
|
|
1160
|
+
});
|
|
1161
|
+
// The chosen name is a guess unless it came from `--ingress-node`: the
|
|
1162
|
+
// cluster-type default reads `--dev` as kind, and `liveNode` re-reads
|
|
1163
|
+
// whatever a previous run wrote. This workload is pinned by `nodeSelector`
|
|
1164
|
+
// with `hostNetwork`, so a name no node carries does not degrade — the pod
|
|
1165
|
+
// stays Pending and every rollout wait times out.
|
|
1166
|
+
const { node: ingressNode, corrected } = Underpost.deploy.resolveSchedulableNode({ node: chosenNode });
|
|
1167
|
+
if (corrected && requestedNode)
|
|
1168
|
+
throw new Error(
|
|
1169
|
+
`[underpost-ingress] --ingress-node ${requestedNode} is not a node in this cluster (schedulable: ${ingressNode})`,
|
|
1170
|
+
);
|
|
1171
|
+
if (corrected)
|
|
1172
|
+
logger.warn('Ingress node does not exist in this cluster; using a schedulable node instead', {
|
|
1173
|
+
chosen: chosenNode,
|
|
1174
|
+
from: liveNode === chosenNode ? 'the live deployment' : 'the cluster-type default',
|
|
1175
|
+
using: ingressNode,
|
|
1176
|
+
});
|
|
1177
|
+
const liveMount = `${
|
|
1178
|
+
shellExec(
|
|
1179
|
+
`kubectl get deployment ${UNDERPOST_INGRESS.name} -n ${namespace} ` +
|
|
1180
|
+
`-o jsonpath='{.spec.template.spec.containers[0].volumeMounts[?(@.name=="nginx-conf")].mountPath}'`,
|
|
1181
|
+
{ stdout: true, silent: true, silentOnError: true },
|
|
1182
|
+
) || ''
|
|
1183
|
+
}`.trim();
|
|
1184
|
+
const liveReady = `${
|
|
1185
|
+
shellExec(
|
|
1186
|
+
`kubectl get deployment ${UNDERPOST_INGRESS.name} -n ${namespace} -o jsonpath='{.status.readyReplicas}'`,
|
|
1187
|
+
{ stdout: true, silent: true, silentOnError: true },
|
|
1188
|
+
) || ''
|
|
1189
|
+
}`.trim();
|
|
1190
|
+
const canHotReload = liveMount === '/etc/underpost-ingress' && parseInt(liveReady || '0', 10) > 0;
|
|
1191
|
+
const changesNode = !!liveNode && ingressNode !== liveNode;
|
|
1192
|
+
const execIngress = (command, execOptions = {}) =>
|
|
1193
|
+
shellExec(`kubectl exec -n ${namespace} deploy/${UNDERPOST_INGRESS.name} -- ${command}`, execOptions);
|
|
1194
|
+
|
|
1195
|
+
if (changesNode) {
|
|
1196
|
+
// Prove the destination can start the exact edge image before Recreate
|
|
1197
|
+
// removes the listener that is serving now. This pod has no hostNetwork,
|
|
1198
|
+
// so it cannot contend for 80/443. On an offline node, IfNotPresent uses
|
|
1199
|
+
// the cache; a missing image fails here while the current edge remains.
|
|
1200
|
+
const preflightPod = `${UNDERPOST_INGRESS.name}-image-preflight`;
|
|
1201
|
+
shellExec(`kubectl delete pod ${preflightPod} -n ${namespace} --ignore-not-found`, {
|
|
1202
|
+
silent: true,
|
|
1203
|
+
silentOnError: true,
|
|
1204
|
+
});
|
|
1205
|
+
try {
|
|
1206
|
+
shellExec(`kubectl apply -f - -n ${namespace} <<'EOF'
|
|
1207
|
+
apiVersion: v1
|
|
1208
|
+
kind: Pod
|
|
1209
|
+
metadata:
|
|
1210
|
+
name: ${preflightPod}
|
|
1211
|
+
namespace: ${namespace}
|
|
1212
|
+
spec:
|
|
1213
|
+
restartPolicy: Never
|
|
1214
|
+
nodeSelector:
|
|
1215
|
+
kubernetes.io/hostname: ${ingressNode}
|
|
1216
|
+
containers:
|
|
1217
|
+
- name: nginx
|
|
1218
|
+
image: ${UNDERPOST_INGRESS.image}
|
|
1219
|
+
imagePullPolicy: IfNotPresent
|
|
1220
|
+
command: ['/bin/sh', '-c', 'nginx -v && sleep 300']
|
|
1221
|
+
EOF
|
|
1222
|
+
`);
|
|
1223
|
+
shellExec(`kubectl wait --for=condition=Ready pod/${preflightPod} -n ${namespace} --timeout=2m`, {
|
|
1224
|
+
silent: true,
|
|
1225
|
+
});
|
|
1226
|
+
} finally {
|
|
1227
|
+
shellExec(`kubectl delete pod ${preflightPod} -n ${namespace} --ignore-not-found`, {
|
|
1228
|
+
silent: true,
|
|
1229
|
+
silentOnError: true,
|
|
1230
|
+
});
|
|
1231
|
+
}
|
|
1232
|
+
}
|
|
1233
|
+
const liveConf = canHotReload
|
|
1234
|
+
? `${
|
|
1235
|
+
execIngress('cat /tmp/nginx.conf', {
|
|
1236
|
+
stdout: true,
|
|
1237
|
+
silent: true,
|
|
1238
|
+
silentOnError: true,
|
|
1239
|
+
}) || ''
|
|
1240
|
+
}`
|
|
1241
|
+
: '';
|
|
1242
|
+
const shouldHotReload = canHotReload && liveConf.trimEnd() !== `${conf}`.trimEnd();
|
|
1243
|
+
|
|
1244
|
+
if (shouldHotReload) {
|
|
1245
|
+
// Validate the exact candidate before either the running master or the
|
|
1246
|
+
// persisted ConfigMap sees it. The quoted heredoc preserves every Nginx
|
|
1247
|
+
// variable and keeps a malformed host table from reaching the edge.
|
|
1248
|
+
shellExec(`kubectl exec -i -n ${namespace} deploy/${UNDERPOST_INGRESS.name} -- sh -c 'cat > /tmp/nginx.candidate.conf' <<'EOF'
|
|
1249
|
+
${conf}
|
|
1250
|
+
EOF
|
|
1251
|
+
`);
|
|
1252
|
+
execIngress('nginx -t -c /tmp/nginx.candidate.conf', { silent: true });
|
|
1253
|
+
try {
|
|
1254
|
+
execIngress(
|
|
1255
|
+
`sh -c 'cp /tmp/nginx.conf /tmp/nginx.previous.conf && ` +
|
|
1256
|
+
`cp /tmp/nginx.candidate.conf /tmp/nginx.conf && nginx -s reload -c /tmp/nginx.conf'`,
|
|
1257
|
+
{ silent: true },
|
|
1258
|
+
);
|
|
1259
|
+
} catch (error) {
|
|
1260
|
+
execIngress(`sh -c 'cp /tmp/nginx.previous.conf /tmp/nginx.conf && nginx -s reload -c /tmp/nginx.conf'`, {
|
|
1261
|
+
silent: true,
|
|
1262
|
+
silentOnError: true,
|
|
1263
|
+
});
|
|
1264
|
+
throw error;
|
|
1265
|
+
}
|
|
1266
|
+
}
|
|
1267
|
+
|
|
1268
|
+
try {
|
|
1269
|
+
shellExec(`kubectl apply -f - -n ${namespace} <<'EOF'
|
|
1270
|
+
${underpostIngressManifestsFactory({
|
|
1271
|
+
namespace,
|
|
1272
|
+
conf,
|
|
1273
|
+
nodeName: ingressNode,
|
|
1274
|
+
})}
|
|
1275
|
+
EOF
|
|
1276
|
+
`);
|
|
1277
|
+
} catch (error) {
|
|
1278
|
+
if (shouldHotReload)
|
|
1279
|
+
execIngress(`sh -c 'cp /tmp/nginx.previous.conf /tmp/nginx.conf && nginx -s reload -c /tmp/nginx.conf'`, {
|
|
1280
|
+
silent: true,
|
|
1281
|
+
silentOnError: true,
|
|
1282
|
+
});
|
|
1283
|
+
throw error;
|
|
1284
|
+
}
|
|
1285
|
+
if (shouldHotReload)
|
|
1286
|
+
execIngress('rm -f /tmp/nginx.previous.conf /tmp/nginx.candidate.conf', {
|
|
1287
|
+
silent: true,
|
|
1288
|
+
silentOnError: true,
|
|
1289
|
+
});
|
|
1290
|
+
// A live pod before apply is not proof the desired template rolled out.
|
|
1291
|
+
// In particular, changing nodeSelector uses Recreate and invalidates the
|
|
1292
|
+
// ready replica that made canHotReload true. Do not let the caller delete
|
|
1293
|
+
// the old route kind until the replacement edge is actually Available.
|
|
1294
|
+
if (!canHotReload || changesNode)
|
|
1295
|
+
shellExec(`kubectl rollout status deployment/${UNDERPOST_INGRESS.name} -n ${namespace} --timeout=5m`, {
|
|
1296
|
+
silent: true,
|
|
1297
|
+
});
|
|
1298
|
+
else if (shouldHotReload)
|
|
1299
|
+
// `nginx -s reload` returns after signalling the master. Give it one
|
|
1300
|
+
// scheduling turn to start the new workers before the caller removes
|
|
1301
|
+
// the old stack's route object.
|
|
1302
|
+
shellExec('sleep 1', { silent: true });
|
|
1303
|
+
shellExec(
|
|
1304
|
+
`kubectl wait --for=condition=Available deployment/${UNDERPOST_INGRESS.name} ` + `-n ${namespace} --timeout=5m`,
|
|
1305
|
+
{ silent: true },
|
|
1306
|
+
);
|
|
1307
|
+
execIngress('nginx -t -c /tmp/nginx.conf', { silent: true });
|
|
1308
|
+
logger.info('Underpost ingress applied', {
|
|
1309
|
+
namespace,
|
|
1310
|
+
backends: Object.keys(backends),
|
|
1311
|
+
hosts: entries.length,
|
|
1312
|
+
node: ingressNode,
|
|
1313
|
+
hotReloaded: shouldHotReload,
|
|
1314
|
+
});
|
|
1315
|
+
return true;
|
|
659
1316
|
},
|
|
660
1317
|
|
|
1318
|
+
/**
|
|
1319
|
+
* @method pruneEndpointlessService
|
|
1320
|
+
* @description Removes a Service that resolves to nothing. kube-proxy does
|
|
1321
|
+
* not ignore such a Service — it installs an ICMP port-unreachable REJECT
|
|
1322
|
+
* for every port it publishes, so an orphan is not inert: it actively
|
|
1323
|
+
* refuses connections on those ports, and the refusal looks exactly like
|
|
1324
|
+
* "no server is listening" even while one is.
|
|
1325
|
+
*
|
|
1326
|
+
* Only an endpointless Service is removed, so a healthy one of the same name
|
|
1327
|
+
* is never touched.
|
|
1328
|
+
* @param {string} name - Service name.
|
|
1329
|
+
* @param {string} [namespace] - Namespace to inspect.
|
|
1330
|
+
* @returns {boolean} True when a Service was pruned.
|
|
1331
|
+
* @memberof UnderpostCluster
|
|
1332
|
+
*/
|
|
1333
|
+
pruneEndpointlessService({ name, namespace = 'default' }) {
|
|
1334
|
+
const exists = shellExec(`kubectl get svc ${name} -n ${namespace} -o name`, {
|
|
1335
|
+
stdout: true,
|
|
1336
|
+
silent: true,
|
|
1337
|
+
silentOnError: true,
|
|
1338
|
+
});
|
|
1339
|
+
if (!exists || !`${exists}`.trim()) return false;
|
|
1340
|
+
// EndpointSlice rather than the deprecated Endpoints API.
|
|
1341
|
+
const addresses = shellExec(
|
|
1342
|
+
`kubectl get endpointslice -n ${namespace} -l kubernetes.io/service-name=${name} ` +
|
|
1343
|
+
`-o jsonpath='{.items[*].endpoints[*].addresses[*]}'`,
|
|
1344
|
+
{ stdout: true, silent: true, silentOnError: true },
|
|
1345
|
+
);
|
|
1346
|
+
if (addresses && `${addresses}`.trim()) return false;
|
|
1347
|
+
logger.warn(`Pruning endpointless Service ${namespace}/${name}`, {
|
|
1348
|
+
reason: 'kube-proxy REJECTs every port a Service with no endpoints publishes',
|
|
1349
|
+
});
|
|
1350
|
+
shellExec(`kubectl delete svc ${name} -n ${namespace} --ignore-not-found`);
|
|
1351
|
+
return true;
|
|
1352
|
+
},
|
|
1353
|
+
|
|
1354
|
+
/**
|
|
1355
|
+
* @method resolveCriSocket
|
|
1356
|
+
* @description CLI-facing binding of {@link CriEndpoint.resolveCriSocket}.
|
|
1357
|
+
* @param {object} [options] - Cluster options (`k3s`, `criSocket`).
|
|
1358
|
+
* @returns {string} CRI endpoint URI.
|
|
1359
|
+
* @memberof UnderpostCluster
|
|
1360
|
+
*/
|
|
1361
|
+
resolveCriSocket,
|
|
1362
|
+
|
|
1363
|
+
/**
|
|
1364
|
+
* @method crictlCommandFactory
|
|
1365
|
+
* @description CLI-facing binding of {@link CriEndpoint.crictlCommandFactory}.
|
|
1366
|
+
* @param {string} args - crictl subcommand and arguments (e.g. `pull mongo:latest`).
|
|
1367
|
+
* @param {object} [options] - Cluster options (`k3s`, `criSocket`).
|
|
1368
|
+
* @returns {string} Full shell command.
|
|
1369
|
+
* @memberof UnderpostCluster
|
|
1370
|
+
*/
|
|
1371
|
+
crictlCommandFactory,
|
|
1372
|
+
|
|
661
1373
|
/**
|
|
662
1374
|
* @method config
|
|
663
1375
|
* @description Configures host-level settings required for Kubernetes.
|
|
@@ -671,26 +1383,17 @@ EOF
|
|
|
671
1383
|
config(options = { underpostRoot: '.' }) {
|
|
672
1384
|
const { underpostRoot } = options;
|
|
673
1385
|
console.log('Applying host configuration: SELinux, Docker, Containerd, and Sysctl settings.');
|
|
674
|
-
|
|
675
|
-
shellExec(`sudo setenforce 0`, {
|
|
676
|
-
silentOnError: true,
|
|
677
|
-
});
|
|
678
|
-
shellExec(`sudo sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config`, {
|
|
679
|
-
silentOnError: true,
|
|
680
|
-
});
|
|
1386
|
+
enforceSELinux(['/var/lib/kubelet', '/var/lib/containerd']);
|
|
681
1387
|
|
|
682
1388
|
// Enable and start Docker and Kubelet services
|
|
683
1389
|
shellExec(`sudo systemctl enable --now docker`); // Docker might not be needed for K3s
|
|
684
1390
|
shellExec(`sudo systemctl enable --now kubelet`); // Kubelet might not be needed for K3s (K3s uses its own agent)
|
|
685
1391
|
|
|
686
|
-
// Configure containerd for
|
|
687
|
-
// This is crucial for kubelet/k3s to interact correctly with containerd
|
|
1392
|
+
// Configure containerd for systemd cgroups and SELinux labeling.
|
|
688
1393
|
shellExec(`containerd config default | sudo tee /etc/containerd/config.toml > /dev/null`);
|
|
689
1394
|
shellExec(`sudo sed -i -e "s/SystemdCgroup = false/SystemdCgroup = true/g" /etc/containerd/config.toml`);
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
// `sudo sed -i '/SystemdCgroup = true/a selinux_disabled = true' /etc/containerd/config.toml`,
|
|
693
|
-
// );
|
|
1395
|
+
shellExec(`sudo sed -i -e "s/enable_selinux = false/enable_selinux = true/g" /etc/containerd/config.toml`);
|
|
1396
|
+
runSELinuxCommands([selinuxRestoreconCommandFactory('/etc/containerd')], { execute: shellExec });
|
|
694
1397
|
// Restart docker after containerd config changes. Rocky 9 uses systemctl,
|
|
695
1398
|
// not the legacy service command.
|
|
696
1399
|
shellExec(`sudo systemctl restart docker || sudo service docker restart || true`);
|
|
@@ -721,7 +1424,7 @@ EOF
|
|
|
721
1424
|
* This applies only what K3s genuinely requires, and every step is guarded
|
|
722
1425
|
* so it is a no-op when the relevant tooling is absent (e.g. minimal images
|
|
723
1426
|
* without SELinux userspace):
|
|
724
|
-
* - SELinux
|
|
1427
|
+
* - SELinux remains Enforcing.
|
|
725
1428
|
* - swap off (Kubernetes best practice).
|
|
726
1429
|
* - br_netfilter + bridge/forward sysctls (pod networking).
|
|
727
1430
|
* - inotify limits.
|
|
@@ -736,16 +1439,8 @@ EOF
|
|
|
736
1439
|
// forever (the upstream unit ships TimeoutStartSec=0).
|
|
737
1440
|
shellExec(`if systemctl is-active --quiet firewalld; then sudo systemctl disable --now firewalld; fi`);
|
|
738
1441
|
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
// mode, so this is a best-effort dev convenience, not a hard requirement.
|
|
742
|
-
shellExec(`if command -v setenforce >/dev/null 2>&1; then sudo setenforce 0; fi`, {
|
|
743
|
-
silentOnError: true,
|
|
744
|
-
});
|
|
745
|
-
shellExec(
|
|
746
|
-
`if [ -f /etc/selinux/config ]; then sudo sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config; fi`,
|
|
747
|
-
{ silentOnError: true },
|
|
748
|
-
);
|
|
1442
|
+
enforceSELinux(K3S_SELINUX_PATHS);
|
|
1443
|
+
shareWithContainers(HOST_VOLUME_PATHS);
|
|
749
1444
|
|
|
750
1445
|
// Disable swap. `swapoff -a` is a no-op without swap; the sed only edits
|
|
751
1446
|
// fstab when a swap line is present.
|
|
@@ -889,7 +1584,7 @@ net.ipv4.ip_forward = 1' | sudo tee /etc/sysctl.d/99-k3s.conf > /dev/null`,
|
|
|
889
1584
|
* @param {object} [options]
|
|
890
1585
|
* @param {boolean} [options.all=true] - Remove all unused images, not just dangling ones.
|
|
891
1586
|
* @param {boolean} [options.crictl=false] - Also prune the CRI runtime via crictl.
|
|
892
|
-
* @param {string} [options.criSocket] - Optional
|
|
1587
|
+
* @param {string} [options.criSocket] - Optional CRI endpoint override; otherwise the live runtime is resolved.
|
|
893
1588
|
* @private
|
|
894
1589
|
*/
|
|
895
1590
|
_pruneContainerCaches(options = {}) {
|
|
@@ -906,11 +1601,9 @@ net.ipv4.ip_forward = 1' | sudo tee /etc/sysctl.d/99-k3s.conf > /dev/null`,
|
|
|
906
1601
|
silentOnError: true,
|
|
907
1602
|
});
|
|
908
1603
|
if (options.crictl) {
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
{ silentOnError: true },
|
|
913
|
-
);
|
|
1604
|
+
shellExec(`if command -v crictl >/dev/null 2>&1; then ${crictlCommandFactory('rmi --prune', options)}; fi`, {
|
|
1605
|
+
silentOnError: true,
|
|
1606
|
+
});
|
|
914
1607
|
}
|
|
915
1608
|
Underpost.cluster._unmountOrphanContainerOverlays();
|
|
916
1609
|
},
|
|
@@ -1020,17 +1713,8 @@ fi`);
|
|
|
1020
1713
|
if (options.removeVolumeHostPaths) Underpost.cluster._cleanHostPathPvs();
|
|
1021
1714
|
else logger.info(' -> Skipping (pass --remove-volume-host-paths to enable).');
|
|
1022
1715
|
|
|
1023
|
-
logger.info('Phase 2/7: SELinux
|
|
1024
|
-
|
|
1025
|
-
shellExec(
|
|
1026
|
-
`if [ -f /etc/selinux/config ]; then sudo sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config; fi`,
|
|
1027
|
-
);
|
|
1028
|
-
shellExec(
|
|
1029
|
-
`if command -v restorecon >/dev/null 2>&1 && [ -d /var/lib/kubelet ]; then sudo restorecon -Rv /var/lib/kubelet; fi`,
|
|
1030
|
-
);
|
|
1031
|
-
shellExec(
|
|
1032
|
-
`if command -v restorecon >/dev/null 2>&1 && [ -d /var/lib/containerd ]; then sudo restorecon -Rv /var/lib/containerd; fi`,
|
|
1033
|
-
);
|
|
1716
|
+
logger.info('Phase 2/7: Enforcing SELinux and restoring runtime contexts...');
|
|
1717
|
+
enforceSELinux(['/var/lib/kubelet', '/var/lib/containerd']);
|
|
1034
1718
|
|
|
1035
1719
|
logger.info('Phase 3/7: Stopping host kubelet and container runtimes (kubeadm-scope only)...');
|
|
1036
1720
|
shellExec(`if systemctl is-active --quiet kubelet; then sudo systemctl stop kubelet; fi`);
|
|
@@ -1039,11 +1723,15 @@ fi`);
|
|
|
1039
1723
|
Underpost.cluster._lazyUmountKubeletMounts();
|
|
1040
1724
|
|
|
1041
1725
|
logger.info('Phase 4/7: Killing control-plane processes and running kubeadm reset...');
|
|
1042
|
-
shellExec(`if command -v crictl >/dev/null 2>&1; then
|
|
1726
|
+
shellExec(`if command -v crictl >/dev/null 2>&1; then ${crictlCommandFactory('rm -a -f')}; fi`, {
|
|
1727
|
+
silentOnError: true,
|
|
1728
|
+
});
|
|
1043
1729
|
// Remove CNI config before stopping sandboxes so Calico's CNI delete hook is
|
|
1044
1730
|
// not invoked (the API server is already down and the hook would fail).
|
|
1045
1731
|
shellExec(`sudo rm -rf /etc/cni/net.d/*`);
|
|
1046
|
-
shellExec(`if command -v crictl >/dev/null 2>&1; then
|
|
1732
|
+
shellExec(`if command -v crictl >/dev/null 2>&1; then ${crictlCommandFactory('rmp -a -f')}; fi`, {
|
|
1733
|
+
silentOnError: true,
|
|
1734
|
+
});
|
|
1047
1735
|
shellExec(`if systemctl is-active --quiet etcd; then sudo systemctl stop etcd; fi`);
|
|
1048
1736
|
for (const port of [6443, 10259, 10257, 2379, 2380]) {
|
|
1049
1737
|
shellExec(`if sudo fuser ${port}/tcp >/dev/null 2>&1; then sudo fuser -k ${port}/tcp; fi`);
|
|
@@ -1226,6 +1914,10 @@ EOF`);
|
|
|
1226
1914
|
shellExec(`sudo rm -rf get_helm.sh`);
|
|
1227
1915
|
}
|
|
1228
1916
|
|
|
1917
|
+
// SOPS and Age for Git-native encrypted secret manifests. Owned by UnderpostSecret so the
|
|
1918
|
+
// same idempotent install backs `underpost secret --install-tools`.
|
|
1919
|
+
Underpost.secret.sops.installTooling();
|
|
1920
|
+
|
|
1229
1921
|
// Install snap
|
|
1230
1922
|
shellExec(`sudo yum install -y snapd`);
|
|
1231
1923
|
shellExec(`sudo systemctl enable --now snapd.socket`);
|
|
@@ -1296,10 +1988,8 @@ EOF`);
|
|
|
1296
1988
|
shellExec(`sudo rm -f /etc/sysctl.d/99-k8s-ipforward.conf`);
|
|
1297
1989
|
shellExec(`sudo rm -f /etc/sysctl.d/99-k8s.conf`);
|
|
1298
1990
|
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
// shellExec(`sudo setenforce 1`);
|
|
1302
|
-
// shellExec(`sudo sed -i 's/^SELINUX=permissive$/SELINUX=enforcing/' /etc/selinux/config`);
|
|
1991
|
+
console.log('Keeping SELinux in enforcing mode...');
|
|
1992
|
+
enforceSELinux();
|
|
1303
1993
|
|
|
1304
1994
|
console.log('Uninstall process completed.');
|
|
1305
1995
|
},
|