underpost 3.2.80 → 3.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/.github/workflows/ghpkg.ci.yml +7 -1
  2. package/.github/workflows/pwa-microservices-template-page.cd.yml +1 -16
  3. package/.github/workflows/pwa-microservices-template-test.ci.yml +1 -1
  4. package/.github/workflows/release.cd.yml +1 -9
  5. package/CHANGELOG.md +291 -1
  6. package/CLI-HELP.md +174 -23
  7. package/README.md +5 -2
  8. package/bin/build.js +7 -5
  9. package/bin/deploy.js +19 -17
  10. package/deploy/lib/logging.sh +96 -0
  11. package/deploy/pwa-microservices-template/deploy.sh +72 -0
  12. package/deploy/release/deploy.sh +62 -0
  13. package/docker-compose.yml +1 -1
  14. package/manifests/cronjobs/dd-cron/dd-cron-backup.yaml +5 -1
  15. package/manifests/cronjobs/dd-cron/dd-cron-dns.yaml +1 -1
  16. package/manifests/cronjobs/dd-cron/dd-cron-vultr.yaml +52 -0
  17. package/manifests/deployment/dd-default-development/deployment.yaml +2 -2
  18. package/manifests/deployment/playwright/deployment.yaml +1 -1
  19. package/manifests/mongodb/kustomization.yaml +4 -1
  20. package/manifests/mongodb/statefulset.yaml +4 -0
  21. package/manifests/mongodb/storage-class.yaml +9 -2
  22. package/package.json +19 -19
  23. package/scripts/audit-selinux.sh +64 -0
  24. package/scripts/coverall-test.sh +24 -0
  25. package/scripts/gpu-diag.sh +0 -0
  26. package/scripts/ip-info.sh +0 -0
  27. package/scripts/k3s-node-setup.sh +18 -15
  28. package/scripts/kubeadm-node-setup.sh +12 -23
  29. package/scripts/link-local-underpost-cli.sh +0 -0
  30. package/scripts/lxd-vm-setup.sh +0 -0
  31. package/scripts/maas-nat-firewalld.sh +0 -0
  32. package/scripts/nat-iptables.sh +12 -4
  33. package/scripts/rhel-grpc-setup.sh +0 -0
  34. package/scripts/rocky-kickstart.sh +25 -9
  35. package/scripts/test-monitor.sh +4 -3
  36. package/src/cli/baremetal.js +1 -2
  37. package/src/cli/cloud-init.js +1 -1
  38. package/src/cli/cluster.js +786 -96
  39. package/src/cli/db.js +11 -4
  40. package/src/cli/deploy.js +1698 -177
  41. package/src/cli/docker-compose.js +19 -178
  42. package/src/cli/env.js +1 -1
  43. package/src/cli/image.js +15 -7
  44. package/src/cli/index.js +245 -44
  45. package/src/cli/ipfs.js +82 -11
  46. package/src/cli/lxd.js +1 -1
  47. package/src/cli/monitor.js +2 -2
  48. package/src/cli/release.js +57 -22
  49. package/src/cli/repository.js +12 -10
  50. package/src/cli/run.js +2195 -427
  51. package/src/cli/secrets.js +969 -0
  52. package/src/cli/ssh.js +206 -105
  53. package/src/cli/system.js +26 -13
  54. package/src/cli/test.js +1 -1
  55. package/src/cli/vultr.js +583 -0
  56. package/src/cli/wireguard.js +2125 -0
  57. package/src/client-builder/client-build.js +102 -13
  58. package/src/client-builder/ssr.js +27 -73
  59. package/src/db/mongo/MongoBootstrap.js +295 -54
  60. package/src/db/mongo/MongooseDB.js +51 -32
  61. package/src/index.js +25 -1
  62. package/src/projects/underpost/catalog-underpost.js +4 -1
  63. package/src/server/backup.js +1 -1
  64. package/src/server/conf.js +1216 -168
  65. package/src/server/cri.js +70 -0
  66. package/src/server/cron.js +249 -51
  67. package/src/server/dns.js +100 -6
  68. package/src/server/environment.js +98 -0
  69. package/src/server/forward-proxy.js +549 -0
  70. package/src/server/middlewares.js +56 -1
  71. package/src/server/process.js +0 -1
  72. package/src/server/selinux.js +185 -0
  73. package/src/server/systemd.js +205 -0
  74. package/src/server/underpost-compression.js +186 -0
  75. package/src/server/underpost-gateway.js +1083 -0
  76. package/src/server/underpost-ingress.js +380 -0
  77. package/test/cluster-instances.test.js +435 -0
  78. package/test/deploy-node-placement.test.js +45 -0
  79. package/test/instance-traffic-plan.test.js +710 -0
  80. package/test/selinux.test.js +71 -0
  81. package/test/sops-secret-store.test.js +612 -0
  82. package/test/underpost-gateway.test.js +510 -0
  83. package/test/underpost-ingress.test.js +305 -0
  84. package/test/wireguard-edge.test.js +1177 -0
@@ -4,9 +4,26 @@
4
4
  * @namespace UnderpostCluster
5
5
  */
6
6
 
7
- import { getNpmRootPath } from '../server/conf.js';
7
+ import { clusterTypeFactory, gatewayApiEnabledFactory, resolveReplicaCount } from '../server/conf.js';
8
+ import { getNpmRootPath, HOST_VOLUME_ROOT } from '../server/environment.js';
8
9
  import { loggerFactory } from '../server/logger.js';
9
10
  import { shellExec } from '../server/process.js';
11
+ import { crictlCommandFactory, resolveCriSocket } from '../server/cri.js';
12
+ import {
13
+ runSELinuxCommands,
14
+ selinuxContainerSharedContextCommandsFactory,
15
+ selinuxEnforcingCommandsFactory,
16
+ selinuxPackagesCommandFactory,
17
+ selinuxRestoreconCommandFactory,
18
+ } from '../server/selinux.js';
19
+ import { UNDERPOST_GATEWAY, seedDefaultStatusPage } from '../server/underpost-gateway.js';
20
+ import {
21
+ UNDERPOST_INGRESS,
22
+ gatewayBackendFactory,
23
+ underpostIngressConfFactory,
24
+ underpostIngressHostMapFactory,
25
+ underpostIngressManifestsFactory,
26
+ } from '../server/underpost-ingress.js';
10
27
  import { MONGODB_DEFAULT_REPLICA_COUNT } from '../db/mongo/MongooseDB.js';
11
28
  import { MongoBootstrap } from '../db/mongo/MongoBootstrap.js';
12
29
  import os from 'os';
@@ -15,6 +32,32 @@ import Underpost from '../index.js';
15
32
 
16
33
  const logger = loggerFactory(import.meta);
17
34
 
35
+ const GATEWAY_API_RELEASE = 'v1.5.1';
36
+ const ENVOY_GATEWAY_VERSION = 'v1.8.3';
37
+ const CONTOUR_NAMESPACE = 'projectcontour';
38
+ const KUBEADM_CONTAINER_MOUNT_PATHS = ['/etc/kubernetes', '/var/lib/etcd', '/var/lib/calico'];
39
+ const HOST_VOLUME_PATHS = ['/data', '/opt/local-path-provisioner', HOST_VOLUME_ROOT];
40
+ const K3S_SELINUX_PATHS = [
41
+ '/usr/local/bin/k3s',
42
+ '/etc/rancher',
43
+ '/var/lib/rancher',
44
+ '/var/lib/kubelet',
45
+ '/var/lib/cni',
46
+ ];
47
+
48
+ const shareWithContainers = (paths = []) => {
49
+ for (const path of paths) shellExec(`sudo mkdir -p ${path}`);
50
+ runSELinuxCommands(selinuxContainerSharedContextCommandsFactory(paths), { execute: shellExec });
51
+ };
52
+
53
+ const enforceSELinux = (paths = []) => {
54
+ shellExec(
55
+ `if [ -f /etc/redhat-release ] && command -v dnf >/dev/null 2>&1 && { ! command -v restorecon >/dev/null 2>&1 || ! command -v semanage >/dev/null 2>&1; }; then ${selinuxPackagesCommandFactory()}; fi`,
56
+ );
57
+ const commands = selinuxEnforcingCommandsFactory({ restorePaths: paths });
58
+ runSELinuxCommands(commands, { execute: shellExec });
59
+ };
60
+
18
61
  /**
19
62
  * @class UnderpostCluster
20
63
  * @description Manages Kubernetes cluster initialization, configuration, and component deployment.
@@ -41,9 +84,11 @@ class UnderpostCluster {
41
84
  * @param {boolean} [options.ipfs=false] - Deploy ipfs-cluster statefulset.
42
85
  * @param {boolean} [options.info=false] - Display extensive Kubernetes cluster information.
43
86
  * @param {boolean} [options.certManager=false] - Deploy Cert-Manager for certificate management.
87
+ * @param {boolean} [options.gatewayApi=false] - Install the Gateway API control plane (CRDs, Envoy Gateway, and the GatewayClass generated Gateways attach to). Exposure follows the environment: host network in `--dev`, NodePort otherwise.
88
+ * @param {string} [options.gatewayClass=''] - GatewayClass name to provision; must match the one baked into generated manifests.
44
89
  * @param {boolean} [options.listPods=false] - List Kubernetes pods.
45
90
  * @param {boolean} [options.reset=false] - Perform a comprehensive reset of Kubernetes and container environments.
46
- * @param {boolean} [options.resetMongodb=false] - Perform a targeted reset of MongoDB components without restarting the entire cluster.
91
+ * @param {boolean} [options.resetMongodb=false] - Perform a targeted reset of MongoDB components without restarting the entire cluster. Combined with `--mongodb` it instead wipes the retained volumes as part of that deploy.
47
92
  * @param {boolean} [options.dev=false] - Run in development mode (adjusts paths).
48
93
  * @param {string} [options.nsUse=''] - Set the current kubectl namespace (creates namespace if it doesn't exist).
49
94
  * @param {string} [options.namespace='default'] - Kubernetes namespace for cluster operations.
@@ -118,7 +163,7 @@ class UnderpostCluster {
118
163
 
119
164
  if (options.config) return options.k3s ? Underpost.cluster.configMinimalK3s() : Underpost.cluster.config();
120
165
 
121
- if (options.chown) return Underpost.cluster.chown(options.k3s ? 'k3s' : options.kubeadm ? 'kubeadm' : 'kind');
166
+ if (options.chown) return Underpost.cluster.chown(clusterTypeFactory(options));
122
167
 
123
168
  const npmRoot = getNpmRootPath();
124
169
  const underpostRoot = options.dev ? '.' : `${npmRoot}/underpost`;
@@ -165,9 +210,13 @@ class UnderpostCluster {
165
210
  });
166
211
  }
167
212
 
168
- // Targeted MongoDB-only reset (does not restart the whole node)
169
- if (options.resetMongodb) {
170
- const clusterType = options.k3s ? 'k3s' : options.kubeadm ? 'kubeadm' : 'kind';
213
+ // Targeted MongoDB-only reset (does not restart the whole node). Combined
214
+ // with --mongodb it is a modifier instead: the deploy below wipes the
215
+ // retained volumes before rolling the StatefulSet out. `--reset` cannot
216
+ // serve that purpose — it short-circuits into the whole-node reset above,
217
+ // which is why `initReplicaSet`'s reset branch was unreachable from the CLI.
218
+ if (options.resetMongodb && !options.mongodb) {
219
+ const clusterType = clusterTypeFactory(options);
171
220
  return await MongoBootstrap.reset({
172
221
  namespace: options.namespace,
173
222
  clusterType,
@@ -200,8 +249,11 @@ class UnderpostCluster {
200
249
  // balancer. The platform exposes services explicitly via Project
201
250
  // Contour / Envoy and NodePort services (see --node-port); leaving the
202
251
  // K3s built-ins enabled would bind the same host ports and conflict.
203
- shellExec(`curl -sfL https://get.k3s.io | sh -s - --disable=traefik --disable=servicelb`);
252
+ shellExec(
253
+ `curl -sfL https://get.k3s.io | sh -s - $(if command -v selinuxenabled >/dev/null 2>&1 && selinuxenabled; then printf '%s' '--selinux'; fi) --disable=traefik --disable=servicelb`,
254
+ );
204
255
  logger.info('K3s installation completed.');
256
+ runSELinuxCommands([selinuxRestoreconCommandFactory(K3S_SELINUX_PATHS)], { execute: shellExec });
205
257
 
206
258
  Underpost.cluster.chown('k3s');
207
259
 
@@ -213,22 +265,15 @@ class UnderpostCluster {
213
265
  } else if (options.kubeadm) {
214
266
  Underpost.cluster.config();
215
267
  Underpost.cluster.natSetup({ underpostRoot });
268
+ shareWithContainers([...KUBEADM_CONTAINER_MOUNT_PATHS, ...HOST_VOLUME_PATHS]);
216
269
  logger.info('Initializing Kubeadm control plane...');
217
270
  // Set default values if not provided
218
271
  const podNetworkCidr = options.podNetworkCidr || '192.168.0.0/16';
219
272
  const controlPlaneEndpoint = options.controlPlaneEndpoint || `${os.hostname()}:6443`;
220
273
 
221
- // Initialize kubeadm control plane.
222
- // Use CRI-O socket when available, otherwise fall back to containerd.
223
- const crioSocket = 'unix:///var/run/crio/crio.sock';
224
- const containerdSocket = 'unix:///run/containerd/containerd.sock';
225
- const criSocket =
226
- shellExec(`test -S /var/run/crio/crio.sock && echo crio || echo containerd`, {
227
- stdout: true,
228
- silent: true,
229
- }).trim() === 'crio'
230
- ? crioSocket
231
- : containerdSocket;
274
+ // Initialize kubeadm control plane against whichever CRI runtime the
275
+ // host actually exposes.
276
+ const criSocket = resolveCriSocket(options);
232
277
  shellExec(
233
278
  `sudo kubeadm init --pod-network-cidr=${podNetworkCidr} --control-plane-endpoint="${controlPlaneEndpoint}" --cri-socket=${criSocket}`,
234
279
  );
@@ -253,12 +298,19 @@ class UnderpostCluster {
253
298
  shellExec(
254
299
  `kubectl apply -f https://cdn.jsdelivr.net/gh/rancher/local-path-provisioner@master/deploy/local-path-storage.yaml`,
255
300
  );
301
+
302
+ // The CNI lands asynchronously, and the node stays NotReady until it
303
+ // does. Block here so the deploy steps that follow (MongoDB and the
304
+ // application workloads) schedule onto a node that can actually run
305
+ // them, instead of racing the Calico rollout.
306
+ logger.info('Waiting for the node to become Ready (Calico rollout)...');
307
+ shellExec(`kubectl wait --for=condition=Ready node --all --timeout=300s`);
256
308
  } else {
257
309
  Underpost.cluster.config();
258
310
  Underpost.cluster.natSetup({ underpostRoot });
259
311
  // Kind cluster initialization (default for development)
260
312
  logger.info('Initializing Kind cluster...');
261
- const devReplicaCount = Math.max(Number(options.replicas) || MONGODB_DEFAULT_REPLICA_COUNT, 3);
313
+ const devReplicaCount = resolveReplicaCount(options.replicas, MONGODB_DEFAULT_REPLICA_COUNT);
262
314
  shellExec(`sudo mkdir -p /data/mongodb`);
263
315
  for (let index = 0; index < devReplicaCount; index++) {
264
316
  shellExec(`sudo mkdir -p /data/mongodb/v${index}`);
@@ -310,7 +362,7 @@ class UnderpostCluster {
310
362
  .readFileSync(`${underpostRoot}/manifests/grafana/deployment.yaml`, 'utf8')
311
363
  .replace('{{GF_SERVER_ROOT_URL}}', options.hosts.split(',')[0])}`;
312
364
  console.log(yaml);
313
- shellExec(`kubectl apply -f - -n ${options.namespace} <<EOF
365
+ shellExec(`kubectl apply -f - -n ${options.namespace} <<'EOF'
314
366
  ${yaml}
315
367
  EOF
316
368
  `);
@@ -329,7 +381,7 @@ EOF
329
381
  .join(',')}]`,
330
382
  )}`;
331
383
  console.log(yaml);
332
- shellExec(`kubectl apply -f - -n ${options.namespace} <<EOF
384
+ shellExec(`kubectl apply -f - -n ${options.namespace} <<'EOF'
333
385
  ${yaml}
334
386
  EOF
335
387
  `);
@@ -361,9 +413,14 @@ EOF
361
413
  await Underpost.ipfs.deploy(options, underpostRoot);
362
414
  }
363
415
  if (options.mariadb) {
364
- shellExec(
365
- `sudo kubectl create secret generic mariadb-secret --from-file=username=/home/dd/engine/engine-private/mariadb-username --from-file=password=/home/dd/engine/engine-private/mariadb-password --dry-run=client -o yaml | kubectl apply -f - -n ${options.namespace}`,
366
- );
416
+ // Secrets before workloads: the StatefulSet's secretKeyRef must resolve at pod admission.
417
+ // Encrypted store first; when no manifest exists the secret is seeded from its origin
418
+ // seed path below. A manifest that exists but fails validation raises rather than
419
+ // silently seeding stale credentials.
420
+ if (!Underpost.secret.sops.applyIfPresent('mariadb-secret', options.namespace))
421
+ shellExec(
422
+ `sudo kubectl create secret generic mariadb-secret --from-file=username=/home/dd/engine/engine-private/mariadb-username --from-file=password=/home/dd/engine/engine-private/mariadb-password --dry-run=client -o yaml | kubectl apply -f - -n ${options.namespace}`,
423
+ );
367
424
  shellExec(`kubectl delete statefulset mariadb-statefulset -n ${options.namespace} --ignore-not-found`);
368
425
 
369
426
  if (options.pullImage) Underpost.cluster.pullImage('mariadb:latest', options);
@@ -371,9 +428,10 @@ EOF
371
428
  shellExec(`kubectl apply -k ${underpostRoot}/manifests/mariadb -n ${options.namespace}`);
372
429
  }
373
430
  if (options.mysql) {
374
- shellExec(
375
- `sudo kubectl create secret generic mysql-secret --from-file=username=/home/dd/engine/engine-private/mysql-username --from-file=password=/home/dd/engine/engine-private/mysql-password --dry-run=client -o yaml | kubectl apply -f - -n ${options.namespace}`,
376
- );
431
+ if (!Underpost.secret.sops.applyIfPresent('mysql-secret', options.namespace))
432
+ shellExec(
433
+ `sudo kubectl create secret generic mysql-secret --from-file=username=/home/dd/engine/engine-private/mysql-username --from-file=password=/home/dd/engine/engine-private/mysql-password --dry-run=client -o yaml | kubectl apply -f - -n ${options.namespace}`,
434
+ );
377
435
  shellExec(`sudo mkdir -p /mnt/data`);
378
436
  shellExec(`sudo chmod 777 /mnt/data`);
379
437
  shellExec(`sudo chown -R $(whoami):$(whoami) /mnt/data`);
@@ -381,9 +439,10 @@ EOF
381
439
  }
382
440
  if (options.postgresql) {
383
441
  if (options.pullImage) Underpost.cluster.pullImage('postgres:latest', options);
384
- shellExec(
385
- `sudo kubectl create secret generic postgres-secret --from-file=password=/home/dd/engine/engine-private/postgresql-password --dry-run=client -o yaml | kubectl apply -f - -n ${options.namespace}`,
386
- );
442
+ if (!Underpost.secret.sops.applyIfPresent('postgres-secret', options.namespace))
443
+ shellExec(
444
+ `sudo kubectl create secret generic postgres-secret --from-file=password=/home/dd/engine/engine-private/postgresql-password --dry-run=client -o yaml | kubectl apply -f - -n ${options.namespace}`,
445
+ );
387
446
  shellExec(`kubectl apply -k ${underpostRoot}/manifests/postgresql -n ${options.namespace}`);
388
447
  }
389
448
  if (options.mongodb4) {
@@ -441,13 +500,13 @@ EOF
441
500
  });
442
501
  }
443
502
  } else if (options.mongodb) {
444
- const clusterType = options.k3s ? 'k3s' : options.kubeadm ? 'kubeadm' : 'kind';
503
+ const clusterType = clusterTypeFactory(options);
445
504
  await MongoBootstrap.initReplicaSet({
446
505
  namespace: options.namespace,
447
- replicaCount: Number(options.replicas) || MONGODB_DEFAULT_REPLICA_COUNT,
506
+ replicaCount: resolveReplicaCount(options.replicas, MONGODB_DEFAULT_REPLICA_COUNT),
448
507
  hostList: serviceHostInput,
449
508
  pullImage: options.pullImage,
450
- reset: options.reset,
509
+ reset: options.resetMongodb === true,
451
510
  clusterType,
452
511
  underpostRoot,
453
512
  });
@@ -459,20 +518,160 @@ EOF
459
518
  });
460
519
  }
461
520
 
521
+ // Installing one stack must never break the other. Whichever is already
522
+ // present decides whether this install takes the node's 80/443 for itself
523
+ // or joins the underpost ingress, so it is read before anything is applied.
524
+ const ingressPresence = Underpost.cluster.ingressStackPresence();
525
+ const sharedIngress =
526
+ (options.contour && ingressPresence.gateway) || (options.gatewayApi && ingressPresence.contour);
527
+
462
528
  if (options.contour) {
463
529
  shellExec(
464
530
  `kubectl apply -f https://cdn.jsdelivr.net/gh/projectcontour/contour@release-1.33/examples/render/contour.yaml`,
465
531
  );
532
+ // The NodePort patch belongs to Contour's own namespace: its `app=envoy`
533
+ // selector only matches the Envoy DaemonSet there. Applied anywhere else
534
+ // the Service is born with no endpoints, and kube-proxy then answers
535
+ // EVERY connection to its ports with an ICMP port-unreachable REJECT —
536
+ // including 443 on every local address, which silently breaks whatever
537
+ // else serves HTTPS on the node.
538
+ Underpost.cluster.pruneEndpointlessService({ name: 'envoy', namespace: options.namespace });
539
+ // Contour's render always claims hostPort 80/443. Left in place beside a
540
+ // Gateway API data plane it would DNAT every packet away from it, so the
541
+ // claim is released and both are reached through the underpost ingress instead.
542
+ if (sharedIngress)
543
+ Underpost.cluster.releaseHostPortClaim({ name: 'envoy', namespace: CONTOUR_NAMESPACE, ports: [80, 443] });
466
544
  if (options.kubeadm) {
467
545
  // Envoy service might need NodePort for kubeadm
468
546
  shellExec(
469
- `sudo kubectl apply -f ${underpostRoot}/manifests/envoy-service-nodeport.yaml -n ${options.namespace}`,
547
+ `sudo kubectl apply -f ${underpostRoot}/manifests/envoy-service-nodeport.yaml -n ${CONTOUR_NAMESPACE}`,
470
548
  );
471
549
  }
472
550
  // K3s has a built-in LoadBalancer (Klipper-lb) that can expose services,
473
551
  // so a specific NodePort service might not be needed or can be configured differently.
474
552
  }
475
553
 
554
+ if (options.gatewayApi) {
555
+ // Gateway API stack: the CRDs, the implementation that serves them, and
556
+ // the GatewayClass every generated Gateway attaches to. Envoy Gateway is
557
+ // the implementation because the manifests this repo generates use two of
558
+ // its ClientTrafficPolicy extension, which carries the QUIC/HTTP3
559
+ // listener config. Both versions are pinned so a rebuild provisions the
560
+ // same control plane.
561
+ const env = options.dev ? 'development' : 'production';
562
+ // Envoy Gateway provisions and owns its own data plane Service in
563
+ // envoy-gateway-system. Any hand-managed Service elsewhere that publishes
564
+ // the same ports without endpoints would have kube-proxy reject 80/443
565
+ // out from under it, and any DaemonSet holding hostPort 80/443 would
566
+ // swallow the traffic before the data plane's listener sees it.
567
+ Underpost.cluster.pruneEndpointlessService({ name: 'envoy', namespace: options.namespace });
568
+ // Contour keeps serving beside this stack when it is already installed —
569
+ // it only gives up the node's ports, which the underpost ingress takes over.
570
+ // With no Contour present the claim is removed outright, since a leftover
571
+ // DaemonSet from an uninstalled stack has nothing left to serve.
572
+ if (sharedIngress)
573
+ Underpost.cluster.releaseHostPortClaim({ name: 'envoy', namespace: CONTOUR_NAMESPACE, ports: [80, 443] });
574
+ else Underpost.cluster.pruneHostPortClaim({ name: 'envoy', namespace: CONTOUR_NAMESPACE, ports: [80, 443] });
575
+ shellExec(
576
+ `kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/${GATEWAY_API_RELEASE}/standard-install.yaml`,
577
+ );
578
+ shellExec(
579
+ `helm upgrade --install eg oci://docker.io/envoyproxy/gateway-helm --version ${ENVOY_GATEWAY_VERSION} ` +
580
+ `--namespace envoy-gateway-system --create-namespace`,
581
+ );
582
+ // The GatewayClass is rejected while the webhook is still coming up.
583
+ shellExec(
584
+ `kubectl wait --namespace envoy-gateway-system --for=condition=Available --timeout=5m deployment/envoy-gateway`,
585
+ { silentOnError: true },
586
+ );
587
+ // Validate against the live CRD schema before creating anything. These
588
+ // are vendor CRDs whose optional fields move between releases, and
589
+ // kubectl applies documents independently: without the pre-flight, a
590
+ // field the installed Envoy Gateway does not know leaves a GatewayClass
591
+ // pointing at an EnvoyProxy that never applied.
592
+ const gatewayClassYaml = Underpost.deploy.gatewayClassYamlFactory({
593
+ env,
594
+ options: { ...options, sharedIngress },
595
+ });
596
+ shellExec(`kubectl apply --dry-run=server -f - <<'EOF'
597
+ ${gatewayClassYaml}
598
+ EOF
599
+ `);
600
+ shellExec(`kubectl apply -f - <<'EOF'
601
+ ${gatewayClassYaml}
602
+ EOF
603
+ `);
604
+ // The static utility that serves status pages and intercepted contexts.
605
+ // It belongs to the gateway tier, not to any one deploy: a single Nginx
606
+ // holds every host's documents, and routes reach it as an ordinary
607
+ // backend — which is what removes the 4096-byte direct-response ceiling.
608
+ //
609
+ // The heredoc delimiter is quoted, as it must be for every generated
610
+ // manifest: the values are already substituted by the template literal,
611
+ // so anything the shell expands here is content. This one carries
612
+ // `nginx.conf`, whose `$uri` an unquoted delimiter deletes — leaving a
613
+ // `try_files` that matches nothing and answers every host's status page
614
+ // with the shared default.
615
+ const underpostGatewayYaml = Underpost.deploy.underpostGatewayYamlFactory(options);
616
+ shellExec(`kubectl apply -f - -n ${options.namespace} <<'EOF'
617
+ ${underpostGatewayYaml}
618
+ EOF
619
+ `);
620
+ const underpostGatewayRoot = Underpost.deploy.underpostGatewayRootFactory(options);
621
+ seedDefaultStatusPage(underpostGatewayRoot);
622
+ logger.info('Gateway static utility applied', {
623
+ name: UNDERPOST_GATEWAY.name,
624
+ root: underpostGatewayRoot,
625
+ });
626
+
627
+ logger.info('Gateway API control plane installed', {
628
+ gatewayApiRelease: GATEWAY_API_RELEASE,
629
+ envoyGateway: ENVOY_GATEWAY_VERSION,
630
+ gatewayClass: Underpost.deploy.gatewayApiConfigFactory(options).gatewayClassName,
631
+ env,
632
+ });
633
+ }
634
+
635
+ // Both stacks are now installed, so the node's 80/443 belong to neither of
636
+ // them. Contour was installed first and Envoy Gateway is arriving, or the
637
+ // reverse — either way this is the point where the underpost ingress has to exist,
638
+ // and the data plane that just came up has to stop binding the host.
639
+ if (sharedIngress) {
640
+ if (options.contour && ingressPresence.gateway) {
641
+ // `--contour` against a cluster that already runs Envoy Gateway: its
642
+ // data plane is still on the host network from a single-stack install,
643
+ // so the EnvoyProxy is re-applied to move it behind the underpost ingress.
644
+ const gatewayClassYaml = Underpost.deploy.gatewayClassYamlFactory({
645
+ env: options.dev ? 'development' : 'production',
646
+ options: { ...options, sharedIngress: true },
647
+ });
648
+ shellExec(`kubectl apply -f - <<'EOF'
649
+ ${gatewayClassYaml}
650
+ EOF
651
+ `);
652
+ shellExec(`kubectl rollout status deployment/envoy-gateway -n envoy-gateway-system --timeout=5m`, {
653
+ silentOnError: true,
654
+ });
655
+ }
656
+ // The edge binds the node directly, so it can only come up once both data
657
+ // planes have actually let go — not merely once their templates say so.
658
+ if (!Underpost.cluster.awaitHostPortsFree({ ports: [80, 443] })) {
659
+ logger.error('Skipping the underpost ingress: it would not be able to bind the node ports', {
660
+ hint: 'resolve the holder above, then re-run this command',
661
+ });
662
+ } else {
663
+ Underpost.cluster.installUnderpostIngress({ namespace: options.namespace, options });
664
+ shellExec(
665
+ `kubectl rollout status deployment/${UNDERPOST_INGRESS.name} -n ${options.namespace} --timeout=3m`,
666
+ { silentOnError: true },
667
+ );
668
+ logger.info('Both ingress stacks are live behind the underpost ingress', {
669
+ ingress: UNDERPOST_INGRESS.name,
670
+ note: 'HTTP/3 is served only by the Gateway API data plane; see the underpost-ingress module',
671
+ });
672
+ }
673
+ }
674
+
476
675
  if (options.certManager) {
477
676
  if (!Underpost.kubectl.get('cert-manager').find((p) => p.STATUS === 'Running')) {
478
677
  shellExec(`helm repo add jetstack https://charts.jetstack.io --force-update`);
@@ -637,27 +836,540 @@ EOF
637
836
  `for node in $(kind get nodes); do cat ${tarPath} | docker exec -i $node ctr --namespace=k8s.io images import -; done`,
638
837
  );
639
838
  shellExec(`rm -f ${tarPath}`);
640
- } else if (options.k3s) {
641
- // K3s uses its own embedded containerd socket, not the host-level one
642
- // used by kubeadm/containerd installations.
643
- shellExec(
644
- `sudo env PATH="$PATH:/usr/local/bin:/usr/bin" crictl --runtime-endpoint unix:///run/k3s/containerd/containerd.sock pull ${image}`,
839
+ } else {
840
+ // Kubeadm / K3s: pull directly into the active CRI runtime.
841
+ shellExec(crictlCommandFactory(`pull ${image}`, options));
842
+ }
843
+ },
844
+
845
+ /**
846
+ * @method pruneHostPortClaim
847
+ * @description Removes a DaemonSet that reserves node ports the caller needs.
848
+ * A `hostPort` is not a soft claim: the CNI hostport plugin installs a DNAT
849
+ * that rewrites every packet arriving on that port to the claiming pod,
850
+ * ahead of any process listening on the node. A second ingress stack bound
851
+ * to the same port therefore never sees a single packet, and the symptom is
852
+ * a connection refused by the *other* stack — with nothing in its own logs.
853
+ *
854
+ * Only a DaemonSet that actually claims one of `ports` is removed.
855
+ * @param {string} name - DaemonSet name.
856
+ * @param {string} namespace - Namespace to inspect.
857
+ * @param {Array<number>} ports - Host ports the caller requires.
858
+ * @returns {boolean} True when a DaemonSet was pruned.
859
+ * @memberof UnderpostCluster
860
+ */
861
+ pruneHostPortClaim({ name, namespace, ports = [80, 443] }) {
862
+ const claimed = shellExec(
863
+ `kubectl get daemonset ${name} -n ${namespace} ` +
864
+ `-o jsonpath='{.spec.template.spec.containers[*].ports[*].hostPort}'`,
865
+ { stdout: true, silent: true, silentOnError: true },
866
+ );
867
+ const claimedPorts = `${claimed || ''}`
868
+ .split(/\s+/)
869
+ .map((port) => parseInt(port, 10))
870
+ .filter((port) => !isNaN(port));
871
+ const conflicts = claimedPorts.filter((port) => ports.includes(port));
872
+ if (conflicts.length === 0) return false;
873
+ logger.warn(`Pruning DaemonSet ${namespace}/${name}: it reserves host ports needed by this ingress stack`, {
874
+ conflicts,
875
+ reason: 'the CNI hostport DNAT redirects those ports before any node listener receives them',
876
+ });
877
+ shellExec(`kubectl delete daemonset ${name} -n ${namespace} --ignore-not-found`);
878
+ return true;
879
+ },
880
+
881
+ /**
882
+ * @method releaseHostPortClaim
883
+ * @description Strips a DaemonSet's `hostPort` claims without removing the
884
+ * workload, so it keeps serving through its ClusterIP.
885
+ *
886
+ * The non-destructive half of {@link UnderpostCluster.pruneHostPortClaim}.
887
+ * Deleting the DaemonSet is right when one stack replaces the other, and
888
+ * wrong when both are meant to stay: here the data plane is still wanted,
889
+ * it just must not hold the node's ports any more — the underpost ingress does.
890
+ *
891
+ * Applied as a JSON Patch, one `remove` op per claiming port. A
892
+ * strategic-merge patch cannot express this: `ports` is a list merged by
893
+ * `containerPort`, so a patch that simply omits `hostPort` merges into the
894
+ * existing element and leaves the claim exactly where it was — reported as
895
+ * `patched (no change)` while the port stays held. Removing an object field
896
+ * does not shift array indices, so every op in one patch stays valid.
897
+ * @param {string} name - DaemonSet name.
898
+ * @param {string} namespace - Namespace to inspect.
899
+ * @param {Array<number>} [ports] - Host ports to release.
900
+ * @returns {boolean} True when the claim is gone afterwards.
901
+ * @memberof UnderpostCluster
902
+ */
903
+ releaseHostPortClaim({ name, namespace, ports = [80, 443] }) {
904
+ const readClaims = () => {
905
+ const raw = shellExec(
906
+ `kubectl get daemonset ${name} -n ${namespace} -o jsonpath='{.spec.template.spec.containers}'`,
907
+ { stdout: true, silent: true, silentOnError: true },
645
908
  );
646
- } else if (options.kubeadm) {
647
- // Kubeadm / K3s: use crictl to pull directly into the active CRI runtime.
648
- // crictl is not in sudo's secure_path; pass full PATH through env.
649
- // Point crictl at CRI-O when the socket exists, otherwise fall back to containerd.
650
- const criSock =
651
- shellExec(`test -S /var/run/crio/crio.sock && echo crio || echo containerd`, {
909
+ try {
910
+ const containers = JSON.parse(`${raw || ''}`);
911
+ return Array.isArray(containers) ? containers : null;
912
+ } catch {
913
+ return null;
914
+ }
915
+ };
916
+
917
+ const containers = readClaims();
918
+ if (!containers) return false;
919
+ const operations = [];
920
+ containers.forEach((container, containerIndex) =>
921
+ (container.ports || []).forEach((port, portIndex) => {
922
+ if (port.hostPort === undefined || !ports.includes(port.hostPort)) return;
923
+ operations.push({
924
+ op: 'remove',
925
+ path: `/spec/template/spec/containers/${containerIndex}/ports/${portIndex}/hostPort`,
926
+ });
927
+ }),
928
+ );
929
+ if (operations.length === 0) return false;
930
+
931
+ logger.warn(`Releasing host ports on DaemonSet ${namespace}/${name}: the underpost ingress owns them now`, {
932
+ ports,
933
+ reason:
934
+ 'its ClusterIP keeps the data plane reachable, and the hostPort DNAT would outrank the underpost ingress listener',
935
+ });
936
+ shellExec(`kubectl patch daemonset ${name} -n ${namespace} --type=json -p '${JSON.stringify(operations)}'`);
937
+
938
+ // Verified rather than assumed: this silently did nothing once already,
939
+ // and the failure only surfaced later as an unschedulable edge.
940
+ const remaining = (readClaims() || [])
941
+ .flatMap((container) => container.ports || [])
942
+ .filter((port) => ports.includes(port.hostPort));
943
+ if (remaining.length > 0) {
944
+ logger.error(`Failed to release host ports on DaemonSet ${namespace}/${name}`, {
945
+ stillClaimed: remaining.map((port) => port.hostPort),
946
+ });
947
+ return false;
948
+ }
949
+ shellExec(`kubectl rollout status daemonset/${name} -n ${namespace} --timeout=3m`, { silentOnError: true });
950
+ return true;
951
+ },
952
+
953
+ /**
954
+ * @method awaitHostPortsFree
955
+ * @description Blocks until no known ingress data plane still holds the node's ports.
956
+ *
957
+ * Releasing a claim edits a template; the pod holding the socket goes away
958
+ * only once the rollout completes. Applying the edge before that leaves it
959
+ * either unschedulable or crash-looping on `bind() … Address in use`, and
960
+ * neither failure names the pod that is actually holding the port.
961
+ * @param {Array<number>} [ports] - Host ports that must be free.
962
+ * @param {string} [contourNamespace] - Namespace holding Contour.
963
+ * @param {number} [timeoutMs] - How long to wait.
964
+ * @returns {boolean} True once nothing claims them.
965
+ * @memberof UnderpostCluster
966
+ */
967
+ awaitHostPortsFree({ ports = [80, 443], contourNamespace = CONTOUR_NAMESPACE, timeoutMs = 3 * 60 * 1000 } = {}) {
968
+ const deadline = Date.now() + timeoutMs;
969
+ const holders = () => {
970
+ const found = [];
971
+ const contourPorts = `${
972
+ shellExec(
973
+ `kubectl get daemonset envoy -n ${contourNamespace} ` +
974
+ `-o jsonpath='{.spec.template.spec.containers[*].ports[*].hostPort}'`,
975
+ { stdout: true, silent: true, silentOnError: true },
976
+ ) || ''
977
+ }`
978
+ .split(/\s+/)
979
+ .map((port) => parseInt(port, 10))
980
+ .filter((port) => ports.includes(port));
981
+ if (contourPorts.length > 0) found.push(`${contourNamespace}/envoy hostPort ${contourPorts.join(',')}`);
982
+ // The Gateway API data plane takes the ports through the host network
983
+ // rather than a hostPort, so its claim is the network mode itself.
984
+ const gatewayHostNetwork = `${
985
+ shellExec(
986
+ `kubectl get deployment -n envoy-gateway-system -l app.kubernetes.io/name=envoy ` +
987
+ `-o jsonpath='{.items[*].spec.template.spec.hostNetwork}'`,
988
+ { stdout: true, silent: true, silentOnError: true },
989
+ ) || ''
990
+ }`.trim();
991
+ if (gatewayHostNetwork.includes('true')) found.push('envoy-gateway-system data plane hostNetwork');
992
+ return found;
993
+ };
994
+
995
+ let current = holders();
996
+ while (current.length > 0 && Date.now() < deadline) {
997
+ logger.info('Waiting for the node ports to be released', { ports, heldBy: current });
998
+ shellExec('sleep 5', { silent: true });
999
+ current = holders();
1000
+ }
1001
+ if (current.length > 0) {
1002
+ logger.error('Node ports are still held; the underpost ingress cannot bind them', {
1003
+ ports,
1004
+ heldBy: current,
1005
+ });
1006
+ return false;
1007
+ }
1008
+ return true;
1009
+ },
1010
+
1011
+ /**
1012
+ * @method ingressStackPresence
1013
+ * @description Which ingress data planes are installed right now.
1014
+ *
1015
+ * Read from the cluster rather than from the flags, because the whole point
1016
+ * is to react to what a previous invocation left behind: `--contour` run
1017
+ * against a cluster that already has Envoy Gateway has to behave differently
1018
+ * from `--contour` on an empty one.
1019
+ * @param {string} [contourNamespace] - Namespace holding Contour.
1020
+ * @returns {{contour: boolean, gateway: boolean}} Presence of each stack.
1021
+ * @memberof UnderpostCluster
1022
+ */
1023
+ ingressStackPresence(contourNamespace = CONTOUR_NAMESPACE) {
1024
+ const exists = (cmd) =>
1025
+ `${shellExec(cmd, { stdout: true, silent: true, silentOnError: true }) || ''}`.trim().length > 0;
1026
+ return {
1027
+ contour: exists(`kubectl get daemonset envoy -n ${contourNamespace} -o name`),
1028
+ gateway: exists(`kubectl get deployment envoy-gateway -n envoy-gateway-system -o name`),
1029
+ };
1030
+ },
1031
+
1032
+ /**
1033
+ * @method refreshUnderpostIngress
1034
+ * @description Rebuilds the underpost ingress host table after routes change.
1035
+ *
1036
+ * The table maps each hostname to the data plane that has a route object for
1037
+ * it, so it goes stale the moment a host is published — a hostname added
1038
+ * after the last build falls through to the default backend, which is the
1039
+ * *other* stack, and answers 404 for a workload that is running perfectly.
1040
+ * Every path that publishes a route therefore ends here.
1041
+ *
1042
+ * A no-op when the ingress is not installed, which is the single-stack case:
1043
+ * there is no table to keep, because the one data plane owns the ports.
1044
+ * The rendered config is unchanged by a blue/green promotion — the table is
1045
+ * host to stack, not host to colour — so an ordinary deploy re-applies the
1046
+ * same bytes and nothing restarts.
1047
+ * @param {string} [namespace] - Namespace holding the ingress.
1048
+ * @param {object} [options] - Cluster options. `ingressNode` is the only
1049
+ * placement override; application `node` / `nodeName` flags are ignored.
1050
+ * @returns {boolean} True when the table was rebuilt.
1051
+ * @memberof UnderpostCluster
1052
+ */
1053
+ refreshUnderpostIngress({ namespace = 'default', options = {} } = {}) {
1054
+ const installed = `${
1055
+ shellExec(`kubectl get deployment ${UNDERPOST_INGRESS.name} -n ${namespace} -o name`, {
1056
+ stdout: true,
1057
+ silent: true,
1058
+ silentOnError: true,
1059
+ }) || ''
1060
+ }`.trim();
1061
+ if (!installed) return false;
1062
+ return Underpost.cluster.installUnderpostIngress({ namespace, options });
1063
+ },
1064
+
1065
+ /**
1066
+ * @method installUnderpostIngress
1067
+ * @description Installs or refreshes the underpost ingress in front of both data planes.
1068
+ *
1069
+ * The host table is built from the route objects that actually exist, so a
1070
+ * hostname reaches the stack that describes it no matter which flag was used
1071
+ * last. Rebuilt on every call, which is what keeps it correct after routes
1072
+ * move between stacks.
1073
+ * @param {string} [namespace] - Namespace to deploy the underpost ingress into.
1074
+ * @param {object} [options] - Cluster options. Use `ingressNode` to
1075
+ * explicitly place or recover the public listener.
1076
+ * @returns {boolean} True when the underpost ingress was applied.
1077
+ * @memberof UnderpostCluster
1078
+ */
1079
+ installUnderpostIngress({ namespace = 'default', options = {} } = {}) {
1080
+ const presence = Underpost.cluster.ingressStackPresence();
1081
+ // Each kind names its hosts in its own field, so each is read with its own
1082
+ // jsonpath rather than one expression that happens to tolerate the other's
1083
+ // shape being absent.
1084
+ const HOST_FIELD = { httpproxy: '.spec.virtualhost.fqdn', httproute: '.spec.hostnames[*]' };
1085
+ const hostsOf = (kind) =>
1086
+ `${
1087
+ shellExec(`kubectl get ${kind} -A -o jsonpath='{range .items[*]}{${HOST_FIELD[kind]}}{" "}{end}'`, {
652
1088
  stdout: true,
653
1089
  silent: true,
654
- }).trim() === 'crio'
655
- ? 'unix:///var/run/crio/crio.sock'
656
- : 'unix:///run/containerd/containerd.sock';
657
- shellExec(`sudo env PATH="$PATH:/usr/local/bin:/usr/bin" crictl --runtime-endpoint ${criSock} pull ${image}`);
1090
+ silentOnError: true,
1091
+ }) || ''
1092
+ }`
1093
+ .split(/\s+/)
1094
+ .map((host) => host.trim())
1095
+ .filter(Boolean);
1096
+
1097
+ const backends = {};
1098
+ if (presence.contour) backends.contour = UNDERPOST_INGRESS.backends.contour;
1099
+ if (presence.gateway) {
1100
+ // Envoy Gateway names its own data plane Service, so it is discovered
1101
+ // rather than assumed; without it the underpost ingress would proxy to nothing.
1102
+ const service = `${
1103
+ shellExec(
1104
+ `kubectl get svc -n envoy-gateway-system -l app.kubernetes.io/name=envoy -o jsonpath='{.items[0].metadata.name}'`,
1105
+ { stdout: true, silent: true, silentOnError: true },
1106
+ ) || ''
1107
+ }`.trim();
1108
+ if (service) backends.gateway = gatewayBackendFactory(service);
1109
+ else logger.warn('Envoy Gateway is installed but has provisioned no data plane Service yet');
1110
+ }
1111
+ if (Object.keys(backends).length === 0) {
1112
+ logger.warn('No ingress data plane installed; skipping the underpost ingress');
1113
+ return false;
658
1114
  }
1115
+
1116
+ const { entries, conflicts } = underpostIngressHostMapFactory({
1117
+ contourHosts: presence.contour ? hostsOf('httpproxy') : [],
1118
+ gatewayHosts: presence.gateway ? hostsOf('httproute') : [],
1119
+ // During migration both route kinds deliberately coexist. Point the
1120
+ // host at the destination stack before the old object is deleted; a
1121
+ // fixed Gateway preference would create a 404 window when migrating in
1122
+ // the opposite direction (Gateway API -> Contour).
1123
+ preferred: gatewayApiEnabledFactory(options) ? 'gateway' : 'contour',
1124
+ });
1125
+ if (conflicts.length > 0)
1126
+ logger.warn('Hosts described by both stacks; the destination stack wins until the old route is removed', {
1127
+ conflicts,
1128
+ destination: gatewayApiEnabledFactory(options) ? 'gateway' : 'contour',
1129
+ });
1130
+
1131
+ const conf = underpostIngressConfFactory({
1132
+ entries,
1133
+ backends,
1134
+ resolver: Underpost.deploy.clusterDnsFactory(),
1135
+ defaultBackend: backends.gateway ? 'gateway' : 'contour',
1136
+ });
1137
+ const liveNode = `${
1138
+ shellExec(
1139
+ `kubectl get deployment ${UNDERPOST_INGRESS.name} -n ${namespace} ` +
1140
+ `-o jsonpath='{.spec.template.spec.nodeSelector.kubernetes\\.io/hostname}'`,
1141
+ { stdout: true, silent: true, silentOnError: true },
1142
+ ) || ''
1143
+ }`.trim();
1144
+ // `node` and `nodeName` place application workloads. Reusing either here
1145
+ // relocates the public listener during an ordinary deploy; with Recreate
1146
+ // that first deletes the healthy edge and can leave the whole site on
1147
+ // connection-refused if the destination cannot pull Nginx. Only the
1148
+ // dedicated ingressNode option may move this workload. Every route-table
1149
+ // refresh otherwise preserves the established edge node.
1150
+ const requestedNode = options.ingressNode || '';
1151
+ const chosenNode =
1152
+ requestedNode ||
1153
+ liveNode ||
1154
+ Underpost.deploy.resolveDeployNode({
1155
+ node: '',
1156
+ kind: options.kind,
1157
+ kubeadm: options.kubeadm,
1158
+ k3s: options.k3s,
1159
+ env: options.dev ? 'development' : 'production',
1160
+ });
1161
+ // The chosen name is a guess unless it came from `--ingress-node`: the
1162
+ // cluster-type default reads `--dev` as kind, and `liveNode` re-reads
1163
+ // whatever a previous run wrote. This workload is pinned by `nodeSelector`
1164
+ // with `hostNetwork`, so a name no node carries does not degrade — the pod
1165
+ // stays Pending and every rollout wait times out.
1166
+ const { node: ingressNode, corrected } = Underpost.deploy.resolveSchedulableNode({ node: chosenNode });
1167
+ if (corrected && requestedNode)
1168
+ throw new Error(
1169
+ `[underpost-ingress] --ingress-node ${requestedNode} is not a node in this cluster (schedulable: ${ingressNode})`,
1170
+ );
1171
+ if (corrected)
1172
+ logger.warn('Ingress node does not exist in this cluster; using a schedulable node instead', {
1173
+ chosen: chosenNode,
1174
+ from: liveNode === chosenNode ? 'the live deployment' : 'the cluster-type default',
1175
+ using: ingressNode,
1176
+ });
1177
+ const liveMount = `${
1178
+ shellExec(
1179
+ `kubectl get deployment ${UNDERPOST_INGRESS.name} -n ${namespace} ` +
1180
+ `-o jsonpath='{.spec.template.spec.containers[0].volumeMounts[?(@.name=="nginx-conf")].mountPath}'`,
1181
+ { stdout: true, silent: true, silentOnError: true },
1182
+ ) || ''
1183
+ }`.trim();
1184
+ const liveReady = `${
1185
+ shellExec(
1186
+ `kubectl get deployment ${UNDERPOST_INGRESS.name} -n ${namespace} -o jsonpath='{.status.readyReplicas}'`,
1187
+ { stdout: true, silent: true, silentOnError: true },
1188
+ ) || ''
1189
+ }`.trim();
1190
+ const canHotReload = liveMount === '/etc/underpost-ingress' && parseInt(liveReady || '0', 10) > 0;
1191
+ const changesNode = !!liveNode && ingressNode !== liveNode;
1192
+ const execIngress = (command, execOptions = {}) =>
1193
+ shellExec(`kubectl exec -n ${namespace} deploy/${UNDERPOST_INGRESS.name} -- ${command}`, execOptions);
1194
+
1195
+ if (changesNode) {
1196
+ // Prove the destination can start the exact edge image before Recreate
1197
+ // removes the listener that is serving now. This pod has no hostNetwork,
1198
+ // so it cannot contend for 80/443. On an offline node, IfNotPresent uses
1199
+ // the cache; a missing image fails here while the current edge remains.
1200
+ const preflightPod = `${UNDERPOST_INGRESS.name}-image-preflight`;
1201
+ shellExec(`kubectl delete pod ${preflightPod} -n ${namespace} --ignore-not-found`, {
1202
+ silent: true,
1203
+ silentOnError: true,
1204
+ });
1205
+ try {
1206
+ shellExec(`kubectl apply -f - -n ${namespace} <<'EOF'
1207
+ apiVersion: v1
1208
+ kind: Pod
1209
+ metadata:
1210
+ name: ${preflightPod}
1211
+ namespace: ${namespace}
1212
+ spec:
1213
+ restartPolicy: Never
1214
+ nodeSelector:
1215
+ kubernetes.io/hostname: ${ingressNode}
1216
+ containers:
1217
+ - name: nginx
1218
+ image: ${UNDERPOST_INGRESS.image}
1219
+ imagePullPolicy: IfNotPresent
1220
+ command: ['/bin/sh', '-c', 'nginx -v && sleep 300']
1221
+ EOF
1222
+ `);
1223
+ shellExec(`kubectl wait --for=condition=Ready pod/${preflightPod} -n ${namespace} --timeout=2m`, {
1224
+ silent: true,
1225
+ });
1226
+ } finally {
1227
+ shellExec(`kubectl delete pod ${preflightPod} -n ${namespace} --ignore-not-found`, {
1228
+ silent: true,
1229
+ silentOnError: true,
1230
+ });
1231
+ }
1232
+ }
1233
+ const liveConf = canHotReload
1234
+ ? `${
1235
+ execIngress('cat /tmp/nginx.conf', {
1236
+ stdout: true,
1237
+ silent: true,
1238
+ silentOnError: true,
1239
+ }) || ''
1240
+ }`
1241
+ : '';
1242
+ const shouldHotReload = canHotReload && liveConf.trimEnd() !== `${conf}`.trimEnd();
1243
+
1244
+ if (shouldHotReload) {
1245
+ // Validate the exact candidate before either the running master or the
1246
+ // persisted ConfigMap sees it. The quoted heredoc preserves every Nginx
1247
+ // variable and keeps a malformed host table from reaching the edge.
1248
+ shellExec(`kubectl exec -i -n ${namespace} deploy/${UNDERPOST_INGRESS.name} -- sh -c 'cat > /tmp/nginx.candidate.conf' <<'EOF'
1249
+ ${conf}
1250
+ EOF
1251
+ `);
1252
+ execIngress('nginx -t -c /tmp/nginx.candidate.conf', { silent: true });
1253
+ try {
1254
+ execIngress(
1255
+ `sh -c 'cp /tmp/nginx.conf /tmp/nginx.previous.conf && ` +
1256
+ `cp /tmp/nginx.candidate.conf /tmp/nginx.conf && nginx -s reload -c /tmp/nginx.conf'`,
1257
+ { silent: true },
1258
+ );
1259
+ } catch (error) {
1260
+ execIngress(`sh -c 'cp /tmp/nginx.previous.conf /tmp/nginx.conf && nginx -s reload -c /tmp/nginx.conf'`, {
1261
+ silent: true,
1262
+ silentOnError: true,
1263
+ });
1264
+ throw error;
1265
+ }
1266
+ }
1267
+
1268
+ try {
1269
+ shellExec(`kubectl apply -f - -n ${namespace} <<'EOF'
1270
+ ${underpostIngressManifestsFactory({
1271
+ namespace,
1272
+ conf,
1273
+ nodeName: ingressNode,
1274
+ })}
1275
+ EOF
1276
+ `);
1277
+ } catch (error) {
1278
+ if (shouldHotReload)
1279
+ execIngress(`sh -c 'cp /tmp/nginx.previous.conf /tmp/nginx.conf && nginx -s reload -c /tmp/nginx.conf'`, {
1280
+ silent: true,
1281
+ silentOnError: true,
1282
+ });
1283
+ throw error;
1284
+ }
1285
+ if (shouldHotReload)
1286
+ execIngress('rm -f /tmp/nginx.previous.conf /tmp/nginx.candidate.conf', {
1287
+ silent: true,
1288
+ silentOnError: true,
1289
+ });
1290
+ // A live pod before apply is not proof the desired template rolled out.
1291
+ // In particular, changing nodeSelector uses Recreate and invalidates the
1292
+ // ready replica that made canHotReload true. Do not let the caller delete
1293
+ // the old route kind until the replacement edge is actually Available.
1294
+ if (!canHotReload || changesNode)
1295
+ shellExec(`kubectl rollout status deployment/${UNDERPOST_INGRESS.name} -n ${namespace} --timeout=5m`, {
1296
+ silent: true,
1297
+ });
1298
+ else if (shouldHotReload)
1299
+ // `nginx -s reload` returns after signalling the master. Give it one
1300
+ // scheduling turn to start the new workers before the caller removes
1301
+ // the old stack's route object.
1302
+ shellExec('sleep 1', { silent: true });
1303
+ shellExec(
1304
+ `kubectl wait --for=condition=Available deployment/${UNDERPOST_INGRESS.name} ` + `-n ${namespace} --timeout=5m`,
1305
+ { silent: true },
1306
+ );
1307
+ execIngress('nginx -t -c /tmp/nginx.conf', { silent: true });
1308
+ logger.info('Underpost ingress applied', {
1309
+ namespace,
1310
+ backends: Object.keys(backends),
1311
+ hosts: entries.length,
1312
+ node: ingressNode,
1313
+ hotReloaded: shouldHotReload,
1314
+ });
1315
+ return true;
659
1316
  },
660
1317
 
1318
+ /**
1319
+ * @method pruneEndpointlessService
1320
+ * @description Removes a Service that resolves to nothing. kube-proxy does
1321
+ * not ignore such a Service — it installs an ICMP port-unreachable REJECT
1322
+ * for every port it publishes, so an orphan is not inert: it actively
1323
+ * refuses connections on those ports, and the refusal looks exactly like
1324
+ * "no server is listening" even while one is.
1325
+ *
1326
+ * Only an endpointless Service is removed, so a healthy one of the same name
1327
+ * is never touched.
1328
+ * @param {string} name - Service name.
1329
+ * @param {string} [namespace] - Namespace to inspect.
1330
+ * @returns {boolean} True when a Service was pruned.
1331
+ * @memberof UnderpostCluster
1332
+ */
1333
+ pruneEndpointlessService({ name, namespace = 'default' }) {
1334
+ const exists = shellExec(`kubectl get svc ${name} -n ${namespace} -o name`, {
1335
+ stdout: true,
1336
+ silent: true,
1337
+ silentOnError: true,
1338
+ });
1339
+ if (!exists || !`${exists}`.trim()) return false;
1340
+ // EndpointSlice rather than the deprecated Endpoints API.
1341
+ const addresses = shellExec(
1342
+ `kubectl get endpointslice -n ${namespace} -l kubernetes.io/service-name=${name} ` +
1343
+ `-o jsonpath='{.items[*].endpoints[*].addresses[*]}'`,
1344
+ { stdout: true, silent: true, silentOnError: true },
1345
+ );
1346
+ if (addresses && `${addresses}`.trim()) return false;
1347
+ logger.warn(`Pruning endpointless Service ${namespace}/${name}`, {
1348
+ reason: 'kube-proxy REJECTs every port a Service with no endpoints publishes',
1349
+ });
1350
+ shellExec(`kubectl delete svc ${name} -n ${namespace} --ignore-not-found`);
1351
+ return true;
1352
+ },
1353
+
1354
+ /**
1355
+ * @method resolveCriSocket
1356
+ * @description CLI-facing binding of {@link CriEndpoint.resolveCriSocket}.
1357
+ * @param {object} [options] - Cluster options (`k3s`, `criSocket`).
1358
+ * @returns {string} CRI endpoint URI.
1359
+ * @memberof UnderpostCluster
1360
+ */
1361
+ resolveCriSocket,
1362
+
1363
+ /**
1364
+ * @method crictlCommandFactory
1365
+ * @description CLI-facing binding of {@link CriEndpoint.crictlCommandFactory}.
1366
+ * @param {string} args - crictl subcommand and arguments (e.g. `pull mongo:latest`).
1367
+ * @param {object} [options] - Cluster options (`k3s`, `criSocket`).
1368
+ * @returns {string} Full shell command.
1369
+ * @memberof UnderpostCluster
1370
+ */
1371
+ crictlCommandFactory,
1372
+
661
1373
  /**
662
1374
  * @method config
663
1375
  * @description Configures host-level settings required for Kubernetes.
@@ -671,26 +1383,17 @@ EOF
671
1383
  config(options = { underpostRoot: '.' }) {
672
1384
  const { underpostRoot } = options;
673
1385
  console.log('Applying host configuration: SELinux, Docker, Containerd, and Sysctl settings.');
674
- // Disable SELinux (permissive mode)
675
- shellExec(`sudo setenforce 0`, {
676
- silentOnError: true,
677
- });
678
- shellExec(`sudo sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config`, {
679
- silentOnError: true,
680
- });
1386
+ enforceSELinux(['/var/lib/kubelet', '/var/lib/containerd']);
681
1387
 
682
1388
  // Enable and start Docker and Kubelet services
683
1389
  shellExec(`sudo systemctl enable --now docker`); // Docker might not be needed for K3s
684
1390
  shellExec(`sudo systemctl enable --now kubelet`); // Kubelet might not be needed for K3s (K3s uses its own agent)
685
1391
 
686
- // Configure containerd for SystemdCgroup and explicitly disable SELinux
687
- // This is crucial for kubelet/k3s to interact correctly with containerd
1392
+ // Configure containerd for systemd cgroups and SELinux labeling.
688
1393
  shellExec(`containerd config default | sudo tee /etc/containerd/config.toml > /dev/null`);
689
1394
  shellExec(`sudo sed -i -e "s/SystemdCgroup = false/SystemdCgroup = true/g" /etc/containerd/config.toml`);
690
- // Add a new line to disable SELinux for the runc runtime
691
- // shellExec(
692
- // `sudo sed -i '/SystemdCgroup = true/a selinux_disabled = true' /etc/containerd/config.toml`,
693
- // );
1395
+ shellExec(`sudo sed -i -e "s/enable_selinux = false/enable_selinux = true/g" /etc/containerd/config.toml`);
1396
+ runSELinuxCommands([selinuxRestoreconCommandFactory('/etc/containerd')], { execute: shellExec });
694
1397
  // Restart docker after containerd config changes. Rocky 9 uses systemctl,
695
1398
  // not the legacy service command.
696
1399
  shellExec(`sudo systemctl restart docker || sudo service docker restart || true`);
@@ -721,7 +1424,7 @@ EOF
721
1424
  * This applies only what K3s genuinely requires, and every step is guarded
722
1425
  * so it is a no-op when the relevant tooling is absent (e.g. minimal images
723
1426
  * without SELinux userspace):
724
- * - SELinux permissive (only if SELinux tooling is present).
1427
+ * - SELinux remains Enforcing.
725
1428
  * - swap off (Kubernetes best practice).
726
1429
  * - br_netfilter + bridge/forward sysctls (pod networking).
727
1430
  * - inotify limits.
@@ -736,16 +1439,8 @@ EOF
736
1439
  // forever (the upstream unit ships TimeoutStartSec=0).
737
1440
  shellExec(`if systemctl is-active --quiet firewalld; then sudo systemctl disable --now firewalld; fi`);
738
1441
 
739
- // SELinux → permissive, but only when the tooling exists. Rocky has it;
740
- // minimal LXD images may not. K3s also installs k3s-selinux for enforcing
741
- // mode, so this is a best-effort dev convenience, not a hard requirement.
742
- shellExec(`if command -v setenforce >/dev/null 2>&1; then sudo setenforce 0; fi`, {
743
- silentOnError: true,
744
- });
745
- shellExec(
746
- `if [ -f /etc/selinux/config ]; then sudo sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config; fi`,
747
- { silentOnError: true },
748
- );
1442
+ enforceSELinux(K3S_SELINUX_PATHS);
1443
+ shareWithContainers(HOST_VOLUME_PATHS);
749
1444
 
750
1445
  // Disable swap. `swapoff -a` is a no-op without swap; the sed only edits
751
1446
  // fstab when a swap line is present.
@@ -889,7 +1584,7 @@ net.ipv4.ip_forward = 1' | sudo tee /etc/sysctl.d/99-k3s.conf > /dev/null`,
889
1584
  * @param {object} [options]
890
1585
  * @param {boolean} [options.all=true] - Remove all unused images, not just dangling ones.
891
1586
  * @param {boolean} [options.crictl=false] - Also prune the CRI runtime via crictl.
892
- * @param {string} [options.criSocket] - Optional crictl --runtime-endpoint override.
1587
+ * @param {string} [options.criSocket] - Optional CRI endpoint override; otherwise the live runtime is resolved.
893
1588
  * @private
894
1589
  */
895
1590
  _pruneContainerCaches(options = {}) {
@@ -906,11 +1601,9 @@ net.ipv4.ip_forward = 1' | sudo tee /etc/sysctl.d/99-k3s.conf > /dev/null`,
906
1601
  silentOnError: true,
907
1602
  });
908
1603
  if (options.crictl) {
909
- const ep = options.criSocket ? `--runtime-endpoint ${options.criSocket} ` : '';
910
- shellExec(
911
- `if command -v crictl >/dev/null 2>&1; then sudo env PATH="$PATH:/usr/local/bin:/usr/bin" crictl ${ep}rmi --prune; fi`,
912
- { silentOnError: true },
913
- );
1604
+ shellExec(`if command -v crictl >/dev/null 2>&1; then ${crictlCommandFactory('rmi --prune', options)}; fi`, {
1605
+ silentOnError: true,
1606
+ });
914
1607
  }
915
1608
  Underpost.cluster._unmountOrphanContainerOverlays();
916
1609
  },
@@ -1020,17 +1713,8 @@ fi`);
1020
1713
  if (options.removeVolumeHostPaths) Underpost.cluster._cleanHostPathPvs();
1021
1714
  else logger.info(' -> Skipping (pass --remove-volume-host-paths to enable).');
1022
1715
 
1023
- logger.info('Phase 2/7: SELinux permissive + restore contexts (when present)...');
1024
- shellExec(`if command -v setenforce >/dev/null 2>&1; then sudo setenforce 0; fi`);
1025
- shellExec(
1026
- `if [ -f /etc/selinux/config ]; then sudo sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config; fi`,
1027
- );
1028
- shellExec(
1029
- `if command -v restorecon >/dev/null 2>&1 && [ -d /var/lib/kubelet ]; then sudo restorecon -Rv /var/lib/kubelet; fi`,
1030
- );
1031
- shellExec(
1032
- `if command -v restorecon >/dev/null 2>&1 && [ -d /var/lib/containerd ]; then sudo restorecon -Rv /var/lib/containerd; fi`,
1033
- );
1716
+ logger.info('Phase 2/7: Enforcing SELinux and restoring runtime contexts...');
1717
+ enforceSELinux(['/var/lib/kubelet', '/var/lib/containerd']);
1034
1718
 
1035
1719
  logger.info('Phase 3/7: Stopping host kubelet and container runtimes (kubeadm-scope only)...');
1036
1720
  shellExec(`if systemctl is-active --quiet kubelet; then sudo systemctl stop kubelet; fi`);
@@ -1039,11 +1723,15 @@ fi`);
1039
1723
  Underpost.cluster._lazyUmountKubeletMounts();
1040
1724
 
1041
1725
  logger.info('Phase 4/7: Killing control-plane processes and running kubeadm reset...');
1042
- shellExec(`if command -v crictl >/dev/null 2>&1; then sudo crictl rm -a -f; fi`, { silentOnError: true });
1726
+ shellExec(`if command -v crictl >/dev/null 2>&1; then ${crictlCommandFactory('rm -a -f')}; fi`, {
1727
+ silentOnError: true,
1728
+ });
1043
1729
  // Remove CNI config before stopping sandboxes so Calico's CNI delete hook is
1044
1730
  // not invoked (the API server is already down and the hook would fail).
1045
1731
  shellExec(`sudo rm -rf /etc/cni/net.d/*`);
1046
- shellExec(`if command -v crictl >/dev/null 2>&1; then sudo crictl rmp -a -f; fi`, { silentOnError: true });
1732
+ shellExec(`if command -v crictl >/dev/null 2>&1; then ${crictlCommandFactory('rmp -a -f')}; fi`, {
1733
+ silentOnError: true,
1734
+ });
1047
1735
  shellExec(`if systemctl is-active --quiet etcd; then sudo systemctl stop etcd; fi`);
1048
1736
  for (const port of [6443, 10259, 10257, 2379, 2380]) {
1049
1737
  shellExec(`if sudo fuser ${port}/tcp >/dev/null 2>&1; then sudo fuser -k ${port}/tcp; fi`);
@@ -1226,6 +1914,10 @@ EOF`);
1226
1914
  shellExec(`sudo rm -rf get_helm.sh`);
1227
1915
  }
1228
1916
 
1917
+ // SOPS and Age for Git-native encrypted secret manifests. Owned by UnderpostSecret so the
1918
+ // same idempotent install backs `underpost secret --install-tools`.
1919
+ Underpost.secret.sops.installTooling();
1920
+
1229
1921
  // Install snap
1230
1922
  shellExec(`sudo yum install -y snapd`);
1231
1923
  shellExec(`sudo systemctl enable --now snapd.socket`);
@@ -1296,10 +1988,8 @@ EOF`);
1296
1988
  shellExec(`sudo rm -f /etc/sysctl.d/99-k8s-ipforward.conf`);
1297
1989
  shellExec(`sudo rm -f /etc/sysctl.d/99-k8s.conf`);
1298
1990
 
1299
- // Restore SELinux to enforcing
1300
- console.log('Restoring SELinux to enforcing mode...');
1301
- // shellExec(`sudo setenforce 1`);
1302
- // shellExec(`sudo sed -i 's/^SELINUX=permissive$/SELINUX=enforcing/' /etc/selinux/config`);
1991
+ console.log('Keeping SELinux in enforcing mode...');
1992
+ enforceSELinux();
1303
1993
 
1304
1994
  console.log('Uninstall process completed.');
1305
1995
  },