underpost 3.2.80 → 3.2.90

package/src/cli/cluster.js

@@ -4,9 +4,18 @@
  * @namespace UnderpostCluster
  */
 
-import { getNpmRootPath } from '../server/conf.js';
+import { clusterTypeFactory, gatewayApiEnabledFactory, getNpmRootPath, resolveReplicaCount } from '../server/conf.js';
 import { loggerFactory } from '../server/logger.js';
 import { shellExec } from '../server/process.js';
+import { crictlCommandFactory, resolveCriSocket } from '../server/cri.js';
+import { UNDERPOST_GATEWAY, seedDefaultStatusPage } from '../server/underpost-gateway.js';
+import {
+  UNDERPOST_INGRESS,
+  gatewayBackendFactory,
+  underpostIngressConfFactory,
+  underpostIngressHostMapFactory,
+  underpostIngressManifestsFactory,
+} from '../server/underpost-ingress.js';
 import { MONGODB_DEFAULT_REPLICA_COUNT } from '../db/mongo/MongooseDB.js';
 import { MongoBootstrap } from '../db/mongo/MongoBootstrap.js';
 import os from 'os';
@@ -15,6 +24,18 @@ import Underpost from '../index.js';
 
 const logger = loggerFactory(import.meta);
 
+// Pinned Gateway API control plane. The CRD release and the implementation are
+// upgraded together, and the pairing is not free choice: Envoy Gateway builds
+// against one `sigs.k8s.io/gateway-api` version (v1.8.3 -> v1.5.1) and reads
+// fields the older CRD schemas do not define, which the API
+// server silently strips. Check the implementation's go.mod before moving
+// either pin.
+const GATEWAY_API_RELEASE = 'v1.5.1';
+const ENVOY_GATEWAY_VERSION = 'v1.8.3';
+// Namespace the upstream Contour render deploys into, and the only one where an
+// `app=envoy` selector resolves to its DaemonSet.
+const CONTOUR_NAMESPACE = 'projectcontour';
+
 /**
  * @class UnderpostCluster
  * @description Manages Kubernetes cluster initialization, configuration, and component deployment.
@@ -41,9 +62,11 @@ class UnderpostCluster {
      * @param {boolean} [options.ipfs=false] - Deploy ipfs-cluster statefulset.
      * @param {boolean} [options.info=false] - Display extensive Kubernetes cluster information.
      * @param {boolean} [options.certManager=false] - Deploy Cert-Manager for certificate management.
+     * @param {boolean} [options.gatewayApi=false] - Install the Gateway API control plane (CRDs, Envoy Gateway, and the GatewayClass generated Gateways attach to). Exposure follows the environment: host network in `--dev`, NodePort otherwise.
+     * @param {string} [options.gatewayClass=''] - GatewayClass name to provision; must match the one baked into generated manifests.
      * @param {boolean} [options.listPods=false] - List Kubernetes pods.
      * @param {boolean} [options.reset=false] - Perform a comprehensive reset of Kubernetes and container environments.
-     * @param {boolean} [options.resetMongodb=false] - Perform a targeted reset of MongoDB components without restarting the entire cluster.
+     * @param {boolean} [options.resetMongodb=false] - Perform a targeted reset of MongoDB components without restarting the entire cluster. Combined with `--mongodb` it instead wipes the retained volumes as part of that deploy.
      * @param {boolean} [options.dev=false] - Run in development mode (adjusts paths).
      * @param {string} [options.nsUse=''] - Set the current kubectl namespace (creates namespace if it doesn't exist).
      * @param {string} [options.namespace='default'] - Kubernetes namespace for cluster operations.
@@ -118,7 +141,7 @@ class UnderpostCluster {
 
       if (options.config) return options.k3s ? Underpost.cluster.configMinimalK3s() : Underpost.cluster.config();
 
-      if (options.chown) return Underpost.cluster.chown(options.k3s ? 'k3s' : options.kubeadm ? 'kubeadm' : 'kind');
+      if (options.chown) return Underpost.cluster.chown(clusterTypeFactory(options));
 
       const npmRoot = getNpmRootPath();
       const underpostRoot = options.dev ? '.' : `${npmRoot}/underpost`;
@@ -165,9 +188,13 @@ class UnderpostCluster {
         });
       }
 
-      // Targeted MongoDB-only reset (does not restart the whole node)
-      if (options.resetMongodb) {
-        const clusterType = options.k3s ? 'k3s' : options.kubeadm ? 'kubeadm' : 'kind';
+      // Targeted MongoDB-only reset (does not restart the whole node). Combined
+      // with --mongodb it is a modifier instead: the deploy below wipes the
+      // retained volumes before rolling the StatefulSet out. `--reset` cannot
+      // serve that purpose — it short-circuits into the whole-node reset above,
+      // which is why `initReplicaSet`'s reset branch was unreachable from the CLI.
+      if (options.resetMongodb && !options.mongodb) {
+        const clusterType = clusterTypeFactory(options);
         return await MongoBootstrap.reset({
           namespace: options.namespace,
           clusterType,
@@ -218,17 +245,9 @@ class UnderpostCluster {
           const podNetworkCidr = options.podNetworkCidr || '192.168.0.0/16';
           const controlPlaneEndpoint = options.controlPlaneEndpoint || `${os.hostname()}:6443`;
 
-          // Initialize kubeadm control plane.
-          // Use CRI-O socket when available, otherwise fall back to containerd.
-          const crioSocket = 'unix:///var/run/crio/crio.sock';
-          const containerdSocket = 'unix:///run/containerd/containerd.sock';
-          const criSocket =
-            shellExec(`test -S /var/run/crio/crio.sock && echo crio || echo containerd`, {
-              stdout: true,
-              silent: true,
-            }).trim() === 'crio'
-              ? crioSocket
-              : containerdSocket;
+          // Initialize kubeadm control plane against whichever CRI runtime the
+          // host actually exposes.
+          const criSocket = resolveCriSocket(options);
           shellExec(
             `sudo kubeadm init --pod-network-cidr=${podNetworkCidr} --control-plane-endpoint="${controlPlaneEndpoint}" --cri-socket=${criSocket}`,
           );
@@ -258,7 +277,7 @@ class UnderpostCluster {
           Underpost.cluster.natSetup({ underpostRoot });
           // Kind cluster initialization (default for development)
           logger.info('Initializing Kind cluster...');
-          const devReplicaCount = Math.max(Number(options.replicas) || MONGODB_DEFAULT_REPLICA_COUNT, 3);
+          const devReplicaCount = resolveReplicaCount(options.replicas, MONGODB_DEFAULT_REPLICA_COUNT);
           shellExec(`sudo mkdir -p /data/mongodb`);
           for (let index = 0; index < devReplicaCount; index++) {
             shellExec(`sudo mkdir -p /data/mongodb/v${index}`);
@@ -310,7 +329,7 @@ class UnderpostCluster {
           .readFileSync(`${underpostRoot}/manifests/grafana/deployment.yaml`, 'utf8')
           .replace('{{GF_SERVER_ROOT_URL}}', options.hosts.split(',')[0])}`;
         console.log(yaml);
-        shellExec(`kubectl apply -f - -n ${options.namespace} <<EOF
+        shellExec(`kubectl apply -f - -n ${options.namespace} <<'EOF'
 ${yaml}
 EOF
 `);
@@ -329,7 +348,7 @@ EOF
             .join(',')}]`,
         )}`;
         console.log(yaml);
-        shellExec(`kubectl apply -f - -n ${options.namespace} <<EOF
+        shellExec(`kubectl apply -f - -n ${options.namespace} <<'EOF'
 ${yaml}
 EOF
 `);
@@ -361,9 +380,14 @@ EOF
         await Underpost.ipfs.deploy(options, underpostRoot);
       }
       if (options.mariadb) {
-        shellExec(
-          `sudo kubectl create secret generic mariadb-secret --from-file=username=/home/dd/engine/engine-private/mariadb-username --from-file=password=/home/dd/engine/engine-private/mariadb-password --dry-run=client -o yaml | kubectl apply -f - -n ${options.namespace}`,
-        );
+        // Secrets before workloads: the StatefulSet's secretKeyRef must resolve at pod admission.
+        // Encrypted store first; when no manifest exists the secret is seeded from its origin
+        // seed path below. A manifest that exists but fails validation raises rather than
+        // silently seeding stale credentials.
+        if (!Underpost.secret.sops.applyIfPresent('mariadb-secret', options.namespace))
+          shellExec(
+            `sudo kubectl create secret generic mariadb-secret --from-file=username=/home/dd/engine/engine-private/mariadb-username --from-file=password=/home/dd/engine/engine-private/mariadb-password --dry-run=client -o yaml | kubectl apply -f - -n ${options.namespace}`,
+          );
         shellExec(`kubectl delete statefulset mariadb-statefulset -n ${options.namespace} --ignore-not-found`);
 
         if (options.pullImage) Underpost.cluster.pullImage('mariadb:latest', options);
@@ -371,9 +395,10 @@ EOF
         shellExec(`kubectl apply -k ${underpostRoot}/manifests/mariadb -n ${options.namespace}`);
       }
       if (options.mysql) {
-        shellExec(
-          `sudo kubectl create secret generic mysql-secret --from-file=username=/home/dd/engine/engine-private/mysql-username --from-file=password=/home/dd/engine/engine-private/mysql-password --dry-run=client -o yaml | kubectl apply -f - -n ${options.namespace}`,
-        );
+        if (!Underpost.secret.sops.applyIfPresent('mysql-secret', options.namespace))
+          shellExec(
+            `sudo kubectl create secret generic mysql-secret --from-file=username=/home/dd/engine/engine-private/mysql-username --from-file=password=/home/dd/engine/engine-private/mysql-password --dry-run=client -o yaml | kubectl apply -f - -n ${options.namespace}`,
+          );
         shellExec(`sudo mkdir -p /mnt/data`);
         shellExec(`sudo chmod 777 /mnt/data`);
         shellExec(`sudo chown -R $(whoami):$(whoami) /mnt/data`);
@@ -381,9 +406,10 @@ EOF
       }
       if (options.postgresql) {
         if (options.pullImage) Underpost.cluster.pullImage('postgres:latest', options);
-        shellExec(
-          `sudo kubectl create secret generic postgres-secret --from-file=password=/home/dd/engine/engine-private/postgresql-password --dry-run=client -o yaml | kubectl apply -f - -n ${options.namespace}`,
-        );
+        if (!Underpost.secret.sops.applyIfPresent('postgres-secret', options.namespace))
+          shellExec(
+            `sudo kubectl create secret generic postgres-secret --from-file=password=/home/dd/engine/engine-private/postgresql-password --dry-run=client -o yaml | kubectl apply -f - -n ${options.namespace}`,
+          );
         shellExec(`kubectl apply -k ${underpostRoot}/manifests/postgresql -n ${options.namespace}`);
       }
       if (options.mongodb4) {
@@ -441,13 +467,13 @@ EOF
             });
         }
       } else if (options.mongodb) {
-        const clusterType = options.k3s ? 'k3s' : options.kubeadm ? 'kubeadm' : 'kind';
+        const clusterType = clusterTypeFactory(options);
         await MongoBootstrap.initReplicaSet({
           namespace: options.namespace,
-          replicaCount: Number(options.replicas) || MONGODB_DEFAULT_REPLICA_COUNT,
+          replicaCount: resolveReplicaCount(options.replicas, MONGODB_DEFAULT_REPLICA_COUNT),
           hostList: serviceHostInput,
           pullImage: options.pullImage,
-          reset: options.reset,
+          reset: options.resetMongodb === true,
           clusterType,
           underpostRoot,
         });
@@ -459,20 +485,160 @@ EOF
           });
       }
 
+      // Installing one stack must never break the other. Whichever is already
+      // present decides whether this install takes the node's 80/443 for itself
+      // or joins the underpost ingress, so it is read before anything is applied.
+      const ingressPresence = Underpost.cluster.ingressStackPresence();
+      const sharedIngress =
+        (options.contour && ingressPresence.gateway) || (options.gatewayApi && ingressPresence.contour);
+
       if (options.contour) {
         shellExec(
           `kubectl apply -f https://cdn.jsdelivr.net/gh/projectcontour/contour@release-1.33/examples/render/contour.yaml`,
         );
+        // The NodePort patch belongs to Contour's own namespace: its `app=envoy`
+        // selector only matches the Envoy DaemonSet there. Applied anywhere else
+        // the Service is born with no endpoints, and kube-proxy then answers
+        // EVERY connection to its ports with an ICMP port-unreachable REJECT —
+        // including 443 on every local address, which silently breaks whatever
+        // else serves HTTPS on the node.
+        Underpost.cluster.pruneEndpointlessService({ name: 'envoy', namespace: options.namespace });
+        // Contour's render always claims hostPort 80/443. Left in place beside a
+        // Gateway API data plane it would DNAT every packet away from it, so the
+        // claim is released and both are reached through the underpost ingress instead.
+        if (sharedIngress)
+          Underpost.cluster.releaseHostPortClaim({ name: 'envoy', namespace: CONTOUR_NAMESPACE, ports: [80, 443] });
         if (options.kubeadm) {
           // Envoy service might need NodePort for kubeadm
           shellExec(
-            `sudo kubectl apply -f ${underpostRoot}/manifests/envoy-service-nodeport.yaml -n ${options.namespace}`,
+            `sudo kubectl apply -f ${underpostRoot}/manifests/envoy-service-nodeport.yaml -n ${CONTOUR_NAMESPACE}`,
           );
         }
         // K3s has a built-in LoadBalancer (Klipper-lb) that can expose services,
         // so a specific NodePort service might not be needed or can be configured differently.
       }
 
+      if (options.gatewayApi) {
+        // Gateway API stack: the CRDs, the implementation that serves them, and
+        // the GatewayClass every generated Gateway attaches to. Envoy Gateway is
+        // the implementation because the manifests this repo generates use two of
+        // its ClientTrafficPolicy extension, which carries the QUIC/HTTP3
+        // listener config. Both versions are pinned so a rebuild provisions the
+        // same control plane.
+        const env = options.dev ? 'development' : 'production';
+        // Envoy Gateway provisions and owns its own data plane Service in
+        // envoy-gateway-system. Any hand-managed Service elsewhere that publishes
+        // the same ports without endpoints would have kube-proxy reject 80/443
+        // out from under it, and any DaemonSet holding hostPort 80/443 would
+        // swallow the traffic before the data plane's listener sees it.
+        Underpost.cluster.pruneEndpointlessService({ name: 'envoy', namespace: options.namespace });
+        // Contour keeps serving beside this stack when it is already installed —
+        // it only gives up the node's ports, which the underpost ingress takes over.
+        // With no Contour present the claim is removed outright, since a leftover
+        // DaemonSet from an uninstalled stack has nothing left to serve.
+        if (sharedIngress)
+          Underpost.cluster.releaseHostPortClaim({ name: 'envoy', namespace: CONTOUR_NAMESPACE, ports: [80, 443] });
+        else Underpost.cluster.pruneHostPortClaim({ name: 'envoy', namespace: CONTOUR_NAMESPACE, ports: [80, 443] });
+        shellExec(
+          `kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/${GATEWAY_API_RELEASE}/standard-install.yaml`,
+        );
+        shellExec(
+          `helm upgrade --install eg oci://docker.io/envoyproxy/gateway-helm --version ${ENVOY_GATEWAY_VERSION} ` +
+            `--namespace envoy-gateway-system --create-namespace`,
+        );
+        // The GatewayClass is rejected while the webhook is still coming up.
+        shellExec(
+          `kubectl wait --namespace envoy-gateway-system --for=condition=Available --timeout=5m deployment/envoy-gateway`,
+          { silentOnError: true },
+        );
+        // Validate against the live CRD schema before creating anything. These
+        // are vendor CRDs whose optional fields move between releases, and
+        // kubectl applies documents independently: without the pre-flight, a
+        // field the installed Envoy Gateway does not know leaves a GatewayClass
+        // pointing at an EnvoyProxy that never applied.
+        const gatewayClassYaml = Underpost.deploy.gatewayClassYamlFactory({
+          env,
+          options: { ...options, sharedIngress },
+        });
+        shellExec(`kubectl apply --dry-run=server -f - <<'EOF'
+${gatewayClassYaml}
+EOF
+`);
+        shellExec(`kubectl apply -f - <<'EOF'
+${gatewayClassYaml}
+EOF
+`);
+        // The static utility that serves status pages and intercepted contexts.
+        // It belongs to the gateway tier, not to any one deploy: a single Nginx
+        // holds every host's documents, and routes reach it as an ordinary
+        // backend — which is what removes the 4096-byte direct-response ceiling.
+        //
+        // The heredoc delimiter is quoted, as it must be for every generated
+        // manifest: the values are already substituted by the template literal,
+        // so anything the shell expands here is content. This one carries
+        // `nginx.conf`, whose `$uri` an unquoted delimiter deletes — leaving a
+        // `try_files` that matches nothing and answers every host's status page
+        // with the shared default.
+        const underpostGatewayYaml = Underpost.deploy.underpostGatewayYamlFactory(options);
+        shellExec(`kubectl apply -f - -n ${options.namespace} <<'EOF'
+${underpostGatewayYaml}
+EOF
+`);
+        const underpostGatewayRoot = Underpost.deploy.underpostGatewayRootFactory(options);
+        seedDefaultStatusPage(underpostGatewayRoot);
+        logger.info('Gateway static utility applied', {
+          name: UNDERPOST_GATEWAY.name,
+          root: underpostGatewayRoot,
+        });
+
+        logger.info('Gateway API control plane installed', {
+          gatewayApiRelease: GATEWAY_API_RELEASE,
+          envoyGateway: ENVOY_GATEWAY_VERSION,
+          gatewayClass: Underpost.deploy.gatewayApiConfigFactory(options).gatewayClassName,
+          env,
+        });
+      }
+
+      // Both stacks are now installed, so the node's 80/443 belong to neither of
+      // them. Contour was installed first and Envoy Gateway is arriving, or the
+      // reverse — either way this is the point where the underpost ingress has to exist,
+      // and the data plane that just came up has to stop binding the host.
+      if (sharedIngress) {
+        if (options.contour && ingressPresence.gateway) {
+          // `--contour` against a cluster that already runs Envoy Gateway: its
+          // data plane is still on the host network from a single-stack install,
+          // so the EnvoyProxy is re-applied to move it behind the underpost ingress.
+          const gatewayClassYaml = Underpost.deploy.gatewayClassYamlFactory({
+            env: options.dev ? 'development' : 'production',
+            options: { ...options, sharedIngress: true },
+          });
+          shellExec(`kubectl apply -f - <<'EOF'
+${gatewayClassYaml}
+EOF
+`);
+          shellExec(`kubectl rollout status deployment/envoy-gateway -n envoy-gateway-system --timeout=5m`, {
+            silentOnError: true,
+          });
+        }
+        // The edge binds the node directly, so it can only come up once both data
+        // planes have actually let go — not merely once their templates say so.
+        if (!Underpost.cluster.awaitHostPortsFree({ ports: [80, 443] })) {
+          logger.error('Skipping the underpost ingress: it would not be able to bind the node ports', {
+            hint: 'resolve the holder above, then re-run this command',
+          });
+        } else {
+          Underpost.cluster.installUnderpostIngress({ namespace: options.namespace, options });
+          shellExec(
+            `kubectl rollout status deployment/${UNDERPOST_INGRESS.name} -n ${options.namespace} --timeout=3m`,
+            { silentOnError: true },
+          );
+          logger.info('Both ingress stacks are live behind the underpost ingress', {
+            ingress: UNDERPOST_INGRESS.name,
+            note: 'HTTP/3 is served only by the Gateway API data plane; see the underpost-ingress module',
+          });
+        }
+      }
+
       if (options.certManager) {
         if (!Underpost.kubectl.get('cert-manager').find((p) => p.STATUS === 'Running')) {
           shellExec(`helm repo add jetstack https://charts.jetstack.io --force-update`);
@@ -637,27 +803,540 @@ EOF
           `for node in $(kind get nodes); do cat ${tarPath} | docker exec -i $node ctr --namespace=k8s.io images import -; done`,
         );
         shellExec(`rm -f ${tarPath}`);
-      } else if (options.k3s) {
-        // K3s uses its own embedded containerd socket, not the host-level one
-        // used by kubeadm/containerd installations.
+      } else {
+        // Kubeadm / K3s: pull directly into the active CRI runtime.
+        shellExec(crictlCommandFactory(`pull ${image}`, options));
+      }
+    },
+
+    /**
+     * @method pruneHostPortClaim
+     * @description Removes a DaemonSet that reserves node ports the caller needs.
+     * A `hostPort` is not a soft claim: the CNI hostport plugin installs a DNAT
+     * that rewrites every packet arriving on that port to the claiming pod,
+     * ahead of any process listening on the node. A second ingress stack bound
+     * to the same port therefore never sees a single packet, and the symptom is
+     * a connection refused by the *other* stack — with nothing in its own logs.
+     *
+     * Only a DaemonSet that actually claims one of `ports` is removed.
+     * @param {string} name - DaemonSet name.
+     * @param {string} namespace - Namespace to inspect.
+     * @param {Array<number>} ports - Host ports the caller requires.
+     * @returns {boolean} True when a DaemonSet was pruned.
+     * @memberof UnderpostCluster
+     */
+    pruneHostPortClaim({ name, namespace, ports = [80, 443] }) {
+      const claimed = shellExec(
+        `kubectl get daemonset ${name} -n ${namespace} ` +
+          `-o jsonpath='{.spec.template.spec.containers[*].ports[*].hostPort}'`,
+        { stdout: true, silent: true, silentOnError: true },
+      );
+      const claimedPorts = `${claimed || ''}`
+        .split(/\s+/)
+        .map((port) => parseInt(port, 10))
+        .filter((port) => !isNaN(port));
+      const conflicts = claimedPorts.filter((port) => ports.includes(port));
+      if (conflicts.length === 0) return false;
+      logger.warn(`Pruning DaemonSet ${namespace}/${name}: it reserves host ports needed by this ingress stack`, {
+        conflicts,
+        reason: 'the CNI hostport DNAT redirects those ports before any node listener receives them',
+      });
+      shellExec(`kubectl delete daemonset ${name} -n ${namespace} --ignore-not-found`);
+      return true;
+    },
+
+    /**
+     * @method releaseHostPortClaim
+     * @description Strips a DaemonSet's `hostPort` claims without removing the
+     * workload, so it keeps serving through its ClusterIP.
+     *
+     * The non-destructive half of {@link UnderpostCluster.pruneHostPortClaim}.
+     * Deleting the DaemonSet is right when one stack replaces the other, and
+     * wrong when both are meant to stay: here the data plane is still wanted,
+     * it just must not hold the node's ports any more — the underpost ingress does.
+     *
+     * Applied as a JSON Patch, one `remove` op per claiming port. A
+     * strategic-merge patch cannot express this: `ports` is a list merged by
+     * `containerPort`, so a patch that simply omits `hostPort` merges into the
+     * existing element and leaves the claim exactly where it was — reported as
+     * `patched (no change)` while the port stays held. Removing an object field
+     * does not shift array indices, so every op in one patch stays valid.
+     * @param {string} name - DaemonSet name.
+     * @param {string} namespace - Namespace to inspect.
+     * @param {Array<number>} [ports] - Host ports to release.
+     * @returns {boolean} True when the claim is gone afterwards.
+     * @memberof UnderpostCluster
+     */
+    releaseHostPortClaim({ name, namespace, ports = [80, 443] }) {
+      const readClaims = () => {
+        const raw = shellExec(
+          `kubectl get daemonset ${name} -n ${namespace} -o jsonpath='{.spec.template.spec.containers}'`,
+          { stdout: true, silent: true, silentOnError: true },
+        );
+        try {
+          const containers = JSON.parse(`${raw || ''}`);
+          return Array.isArray(containers) ? containers : null;
+        } catch {
+          return null;
+        }
+      };
+
+      const containers = readClaims();
+      if (!containers) return false;
+      const operations = [];
+      containers.forEach((container, containerIndex) =>
+        (container.ports || []).forEach((port, portIndex) => {
+          if (port.hostPort === undefined || !ports.includes(port.hostPort)) return;
+          operations.push({
+            op: 'remove',
+            path: `/spec/template/spec/containers/${containerIndex}/ports/${portIndex}/hostPort`,
+          });
+        }),
+      );
+      if (operations.length === 0) return false;
+
+      logger.warn(`Releasing host ports on DaemonSet ${namespace}/${name}: the underpost ingress owns them now`, {
+        ports,
+        reason:
+          'its ClusterIP keeps the data plane reachable, and the hostPort DNAT would outrank the underpost ingress listener',
+      });
+      shellExec(`kubectl patch daemonset ${name} -n ${namespace} --type=json -p '${JSON.stringify(operations)}'`);
+
+      // Verified rather than assumed: this silently did nothing once already,
+      // and the failure only surfaced later as an unschedulable edge.
+      const remaining = (readClaims() || [])
+        .flatMap((container) => container.ports || [])
+        .filter((port) => ports.includes(port.hostPort));
+      if (remaining.length > 0) {
+        logger.error(`Failed to release host ports on DaemonSet ${namespace}/${name}`, {
+          stillClaimed: remaining.map((port) => port.hostPort),
+        });
+        return false;
+      }
+      shellExec(`kubectl rollout status daemonset/${name} -n ${namespace} --timeout=3m`, { silentOnError: true });
+      return true;
+    },
+
+    /**
+     * @method awaitHostPortsFree
+     * @description Blocks until no known ingress data plane still holds the node's ports.
+     *
+     * Releasing a claim edits a template; the pod holding the socket goes away
+     * only once the rollout completes. Applying the edge before that leaves it
+     * either unschedulable or crash-looping on `bind() … Address in use`, and
+     * neither failure names the pod that is actually holding the port.
+     * @param {Array<number>} [ports] - Host ports that must be free.
+     * @param {string} [contourNamespace] - Namespace holding Contour.
+     * @param {number} [timeoutMs] - How long to wait.
+     * @returns {boolean} True once nothing claims them.
+     * @memberof UnderpostCluster
+     */
+    awaitHostPortsFree({ ports = [80, 443], contourNamespace = CONTOUR_NAMESPACE, timeoutMs = 3 * 60 * 1000 } = {}) {
+      const deadline = Date.now() + timeoutMs;
+      const holders = () => {
+        const found = [];
+        const contourPorts = `${
+          shellExec(
+            `kubectl get daemonset envoy -n ${contourNamespace} ` +
+              `-o jsonpath='{.spec.template.spec.containers[*].ports[*].hostPort}'`,
+            { stdout: true, silent: true, silentOnError: true },
+          ) || ''
+        }`
+          .split(/\s+/)
+          .map((port) => parseInt(port, 10))
+          .filter((port) => ports.includes(port));
+        if (contourPorts.length > 0) found.push(`${contourNamespace}/envoy hostPort ${contourPorts.join(',')}`);
+        // The Gateway API data plane takes the ports through the host network
+        // rather than a hostPort, so its claim is the network mode itself.
+        const gatewayHostNetwork = `${
+          shellExec(
+            `kubectl get deployment -n envoy-gateway-system -l app.kubernetes.io/name=envoy ` +
+              `-o jsonpath='{.items[*].spec.template.spec.hostNetwork}'`,
+            { stdout: true, silent: true, silentOnError: true },
+          ) || ''
+        }`.trim();
+        if (gatewayHostNetwork.includes('true')) found.push('envoy-gateway-system data plane hostNetwork');
+        return found;
+      };
+
+      let current = holders();
+      while (current.length > 0 && Date.now() < deadline) {
+        logger.info('Waiting for the node ports to be released', { ports, heldBy: current });
+        shellExec('sleep 5', { silent: true });
+        current = holders();
+      }
+      if (current.length > 0) {
+        logger.error('Node ports are still held; the underpost ingress cannot bind them', {
+          ports,
+          heldBy: current,
+        });
+        return false;
+      }
+      return true;
+    },
+
+    /**
+     * @method ingressStackPresence
+     * @description Which ingress data planes are installed right now.
+     *
+     * Read from the cluster rather than from the flags, because the whole point
+     * is to react to what a previous invocation left behind: `--contour` run
+     * against a cluster that already has Envoy Gateway has to behave differently
+     * from `--contour` on an empty one.
+     * @param {string} [contourNamespace] - Namespace holding Contour.
+     * @returns {{contour: boolean, gateway: boolean}} Presence of each stack.
+     * @memberof UnderpostCluster
+     */
+    ingressStackPresence(contourNamespace = CONTOUR_NAMESPACE) {
+      const exists = (cmd) =>
+        `${shellExec(cmd, { stdout: true, silent: true, silentOnError: true }) || ''}`.trim().length > 0;
+      return {
+        contour: exists(`kubectl get daemonset envoy -n ${contourNamespace} -o name`),
+        gateway: exists(`kubectl get deployment envoy-gateway -n envoy-gateway-system -o name`),
+      };
+    },
+
+    /**
+     * @method refreshUnderpostIngress
+     * @description Rebuilds the underpost ingress host table after routes change.
+     *
+     * The table maps each hostname to the data plane that has a route object for
+     * it, so it goes stale the moment a host is published — a hostname added
+     * after the last build falls through to the default backend, which is the
+     * *other* stack, and answers 404 for a workload that is running perfectly.
+     * Every path that publishes a route therefore ends here.
+     *
+     * A no-op when the ingress is not installed, which is the single-stack case:
+     * there is no table to keep, because the one data plane owns the ports.
+     * The rendered config is unchanged by a blue/green promotion — the table is
+     * host to stack, not host to colour — so an ordinary deploy re-applies the
+     * same bytes and nothing restarts.
+     * @param {string} [namespace] - Namespace holding the ingress.
+     * @param {object} [options] - Cluster options. `ingressNode` is the only
+     * placement override; application `node` / `nodeName` flags are ignored.
+     * @returns {boolean} True when the table was rebuilt.
+     * @memberof UnderpostCluster
+     */
+    refreshUnderpostIngress({ namespace = 'default', options = {} } = {}) {
+      const installed = `${
+        shellExec(`kubectl get deployment ${UNDERPOST_INGRESS.name} -n ${namespace} -o name`, {
+          stdout: true,
+          silent: true,
+          silentOnError: true,
+        }) || ''
+      }`.trim();
+      if (!installed) return false;
+      return Underpost.cluster.installUnderpostIngress({ namespace, options });
+    },
+
+    /**
+     * @method installUnderpostIngress
+     * @description Installs or refreshes the underpost ingress in front of both data planes.
+     *
+     * The host table is built from the route objects that actually exist, so a
+     * hostname reaches the stack that describes it no matter which flag was used
+     * last. Rebuilt on every call, which is what keeps it correct after routes
+     * move between stacks.
+     * @param {string} [namespace] - Namespace to deploy the underpost ingress into.
+     * @param {object} [options] - Cluster options. Use `ingressNode` to
+     * explicitly place or recover the public listener.
+     * @returns {boolean} True when the underpost ingress was applied.
+     * @memberof UnderpostCluster
+     */
+    installUnderpostIngress({ namespace = 'default', options = {} } = {}) {
+      const presence = Underpost.cluster.ingressStackPresence();
+      // Each kind names its hosts in its own field, so each is read with its own
+      // jsonpath rather than one expression that happens to tolerate the other's
+      // shape being absent.
+      const HOST_FIELD = { httpproxy: '.spec.virtualhost.fqdn', httproute: '.spec.hostnames[*]' };
+      const hostsOf = (kind) =>
+        `${
+          shellExec(`kubectl get ${kind} -A -o jsonpath='{range .items[*]}{${HOST_FIELD[kind]}}{" "}{end}'`, {
+            stdout: true,
+            silent: true,
+            silentOnError: true,
+          }) || ''
+        }`
+          .split(/\s+/)
+          .map((host) => host.trim())
+          .filter(Boolean);
+
+      const backends = {};
+      if (presence.contour) backends.contour = UNDERPOST_INGRESS.backends.contour;
+      if (presence.gateway) {
+        // Envoy Gateway names its own data plane Service, so it is discovered
+        // rather than assumed; without it the underpost ingress would proxy to nothing.
+        const service = `${
+          shellExec(
+            `kubectl get svc -n envoy-gateway-system -l app.kubernetes.io/name=envoy -o jsonpath='{.items[0].metadata.name}'`,
+            { stdout: true, silent: true, silentOnError: true },
+          ) || ''
+        }`.trim();
+        if (service) backends.gateway = gatewayBackendFactory(service);
+        else logger.warn('Envoy Gateway is installed but has provisioned no data plane Service yet');
+      }
+      if (Object.keys(backends).length === 0) {
+        logger.warn('No ingress data plane installed; skipping the underpost ingress');
+        return false;
+      }
+
+      const { entries, conflicts } = underpostIngressHostMapFactory({
+        contourHosts: presence.contour ? hostsOf('httpproxy') : [],
+        gatewayHosts: presence.gateway ? hostsOf('httproute') : [],
+        // During migration both route kinds deliberately coexist. Point the
+        // host at the destination stack before the old object is deleted; a
+        // fixed Gateway preference would create a 404 window when migrating in
+        // the opposite direction (Gateway API -> Contour).
+        preferred: gatewayApiEnabledFactory(options) ? 'gateway' : 'contour',
+      });
+      if (conflicts.length > 0)
+        logger.warn('Hosts described by both stacks; the destination stack wins until the old route is removed', {
+          conflicts,
+          destination: gatewayApiEnabledFactory(options) ? 'gateway' : 'contour',
+        });
+
+      const conf = underpostIngressConfFactory({
+        entries,
+        backends,
+        resolver: Underpost.deploy.clusterDnsFactory(),
+        defaultBackend: backends.gateway ? 'gateway' : 'contour',
+      });
+      const liveNode = `${
         shellExec(
-          `sudo env PATH="$PATH:/usr/local/bin:/usr/bin" crictl --runtime-endpoint unix:///run/k3s/containerd/containerd.sock pull ${image}`,
+          `kubectl get deployment ${UNDERPOST_INGRESS.name} -n ${namespace} ` +
+            `-o jsonpath='{.spec.template.spec.nodeSelector.kubernetes\\.io/hostname}'`,
+          { stdout: true, silent: true, silentOnError: true },
+        ) || ''
+      }`.trim();
+      // `node` and `nodeName` place application workloads. Reusing either here
+      // relocates the public listener during an ordinary deploy; with Recreate
+      // that first deletes the healthy edge and can leave the whole site on
+      // connection-refused if the destination cannot pull Nginx. Only the
+      // dedicated ingressNode option may move this workload. Every route-table
+      // refresh otherwise preserves the established edge node.
+      const requestedNode = options.ingressNode || '';
+      const chosenNode =
+        requestedNode ||
+        liveNode ||
+        Underpost.deploy.resolveDeployNode({
+          node: '',
+          kind: options.kind,
+          kubeadm: options.kubeadm,
+          k3s: options.k3s,
+          env: options.dev ? 'development' : 'production',
+        });
+      // The chosen name is a guess unless it came from `--ingress-node`: the
+      // cluster-type default reads `--dev` as kind, and `liveNode` re-reads
+      // whatever a previous run wrote. This workload is pinned by `nodeSelector`
+      // with `hostNetwork`, so a name no node carries does not degrade — the pod
+      // stays Pending and every rollout wait times out.
+      const { node: ingressNode, corrected } = Underpost.deploy.resolveSchedulableNode({ node: chosenNode });
+      if (corrected && requestedNode)
+        throw new Error(
+          `[underpost-ingress] --ingress-node ${requestedNode} is not a node in this cluster (schedulable: ${ingressNode})`,
         );
-      } else if (options.kubeadm) {
-        // Kubeadm / K3s: use crictl to pull directly into the active CRI runtime.
-        // crictl is not in sudo's secure_path; pass full PATH through env.
-        // Point crictl at CRI-O when the socket exists, otherwise fall back to containerd.
-        const criSock =
-          shellExec(`test -S /var/run/crio/crio.sock && echo crio || echo containerd`, {
+      if (corrected)
+        logger.warn('Ingress node does not exist in this cluster; using a schedulable node instead', {
+          chosen: chosenNode,
+          from: liveNode === chosenNode ? 'the live deployment' : 'the cluster-type default',
+          using: ingressNode,
+        });
+      const liveMount = `${
+        shellExec(
+          `kubectl get deployment ${UNDERPOST_INGRESS.name} -n ${namespace} ` +
+            `-o jsonpath='{.spec.template.spec.containers[0].volumeMounts[?(@.name=="nginx-conf")].mountPath}'`,
+          { stdout: true, silent: true, silentOnError: true },
+        ) || ''
+      }`.trim();
+      const liveReady = `${
+        shellExec(
+          `kubectl get deployment ${UNDERPOST_INGRESS.name} -n ${namespace} -o jsonpath='{.status.readyReplicas}'`,
+          { stdout: true, silent: true, silentOnError: true },
+        ) || ''
+      }`.trim();
+      const canHotReload = liveMount === '/etc/underpost-ingress' && parseInt(liveReady || '0', 10) > 0;
+      const changesNode = !!liveNode && ingressNode !== liveNode;
+      const execIngress = (command, execOptions = {}) =>
+        shellExec(`kubectl exec -n ${namespace} deploy/${UNDERPOST_INGRESS.name} -- ${command}`, execOptions);
+
+      if (changesNode) {
+        // Prove the destination can start the exact edge image before Recreate
+        // removes the listener that is serving now. This pod has no hostNetwork,
+        // so it cannot contend for 80/443. On an offline node, IfNotPresent uses
+        // the cache; a missing image fails here while the current edge remains.
+        const preflightPod = `${UNDERPOST_INGRESS.name}-image-preflight`;
+        shellExec(`kubectl delete pod ${preflightPod} -n ${namespace} --ignore-not-found`, {
+          silent: true,
+          silentOnError: true,
+        });
+        try {
+          shellExec(`kubectl apply -f - -n ${namespace} <<'EOF'
+apiVersion: v1
+kind: Pod
+metadata:
+  name: ${preflightPod}
+  namespace: ${namespace}
+spec:
+  restartPolicy: Never
+  nodeSelector:
+    kubernetes.io/hostname: ${ingressNode}
+  containers:
+    - name: nginx
+      image: ${UNDERPOST_INGRESS.image}
+      imagePullPolicy: IfNotPresent
+      command: ['/bin/sh', '-c', 'nginx -v && sleep 300']
+EOF
+`);
+          shellExec(`kubectl wait --for=condition=Ready pod/${preflightPod} -n ${namespace} --timeout=2m`, {
+            silent: true,
+          });
+        } finally {
+          shellExec(`kubectl delete pod ${preflightPod} -n ${namespace} --ignore-not-found`, {
+            silent: true,
+            silentOnError: true,
+          });
+        }
+      }
+      const liveConf = canHotReload
+        ? `${execIngress('cat /tmp/nginx.conf', {
             stdout: true,
             silent: true,
-          }).trim() === 'crio'
-            ? 'unix:///var/run/crio/crio.sock'
-            : 'unix:///run/containerd/containerd.sock';
-        shellExec(`sudo env PATH="$PATH:/usr/local/bin:/usr/bin" crictl --runtime-endpoint ${criSock} pull ${image}`);
+            silentOnError: true,
+          }) || ''}`
+        : '';
+      const shouldHotReload = canHotReload && liveConf.trimEnd() !== `${conf}`.trimEnd();
+
+      if (shouldHotReload) {
+        // Validate the exact candidate before either the running master or the
+        // persisted ConfigMap sees it. The quoted heredoc preserves every Nginx
+        // variable and keeps a malformed host table from reaching the edge.
+        shellExec(`kubectl exec -i -n ${namespace} deploy/${UNDERPOST_INGRESS.name} -- sh -c 'cat > /tmp/nginx.candidate.conf' <<'EOF'
+${conf}
+EOF
+`);
+        execIngress('nginx -t -c /tmp/nginx.candidate.conf', { silent: true });
+        try {
+          execIngress(
+            `sh -c 'cp /tmp/nginx.conf /tmp/nginx.previous.conf && ` +
+              `cp /tmp/nginx.candidate.conf /tmp/nginx.conf && nginx -s reload -c /tmp/nginx.conf'`,
+            { silent: true },
+          );
+        } catch (error) {
+          execIngress(
+            `sh -c 'cp /tmp/nginx.previous.conf /tmp/nginx.conf && nginx -s reload -c /tmp/nginx.conf'`,
+            { silent: true, silentOnError: true },
+          );
+          throw error;
+        }
       }
+
+      try {
+        shellExec(`kubectl apply -f - -n ${namespace} <<'EOF'
+${underpostIngressManifestsFactory({
+  namespace,
+  conf,
+  nodeName: ingressNode,
+})}
+EOF
+`);
+      } catch (error) {
+        if (shouldHotReload)
+          execIngress(
+            `sh -c 'cp /tmp/nginx.previous.conf /tmp/nginx.conf && nginx -s reload -c /tmp/nginx.conf'`,
+            { silent: true, silentOnError: true },
+          );
+        throw error;
+      }
+      if (shouldHotReload)
+        execIngress('rm -f /tmp/nginx.previous.conf /tmp/nginx.candidate.conf', {
+          silent: true,
+          silentOnError: true,
+        });
+      // A live pod before apply is not proof the desired template rolled out.
+      // In particular, changing nodeSelector uses Recreate and invalidates the
+      // ready replica that made canHotReload true. Do not let the caller delete
+      // the old route kind until the replacement edge is actually Available.
+      if (!canHotReload || changesNode)
+        shellExec(
+          `kubectl rollout status deployment/${UNDERPOST_INGRESS.name} -n ${namespace} --timeout=5m`,
+          { silent: true },
+        );
+      else if (shouldHotReload)
+        // `nginx -s reload` returns after signalling the master. Give it one
+        // scheduling turn to start the new workers before the caller removes
+        // the old stack's route object.
+        shellExec('sleep 1', { silent: true });
+      shellExec(
+        `kubectl wait --for=condition=Available deployment/${UNDERPOST_INGRESS.name} ` +
+          `-n ${namespace} --timeout=5m`,
+        { silent: true },
+      );
+      execIngress('nginx -t -c /tmp/nginx.conf', { silent: true });
+      logger.info('Underpost ingress applied', {
+        namespace,
+        backends: Object.keys(backends),
+        hosts: entries.length,
+        node: ingressNode,
+        hotReloaded: shouldHotReload,
+      });
+      return true;
+    },
+
+    /**
+     * @method pruneEndpointlessService
+     * @description Removes a Service that resolves to nothing. kube-proxy does
+     * not ignore such a Service — it installs an ICMP port-unreachable REJECT
+     * for every port it publishes, so an orphan is not inert: it actively
+     * refuses connections on those ports, and the refusal looks exactly like
+     * "no server is listening" even while one is.
+     *
+     * Only an endpointless Service is removed, so a healthy one of the same name
+     * is never touched.
+     * @param {string} name - Service name.
+     * @param {string} [namespace] - Namespace to inspect.
+     * @returns {boolean} True when a Service was pruned.
+     * @memberof UnderpostCluster
+     */
+    pruneEndpointlessService({ name, namespace = 'default' }) {
+      const exists = shellExec(`kubectl get svc ${name} -n ${namespace} -o name`, {
+        stdout: true,
+        silent: true,
+        silentOnError: true,
+      });
+      if (!exists || !`${exists}`.trim()) return false;
+      // EndpointSlice rather than the deprecated Endpoints API.
+      const addresses = shellExec(
+        `kubectl get endpointslice -n ${namespace} -l kubernetes.io/service-name=${name} ` +
+          `-o jsonpath='{.items[*].endpoints[*].addresses[*]}'`,
+        { stdout: true, silent: true, silentOnError: true },
+      );
+      if (addresses && `${addresses}`.trim()) return false;
+      logger.warn(`Pruning endpointless Service ${namespace}/${name}`, {
+        reason: 'kube-proxy REJECTs every port a Service with no endpoints publishes',
+      });
+      shellExec(`kubectl delete svc ${name} -n ${namespace} --ignore-not-found`);
+      return true;
     },
 
+    /**
+     * @method resolveCriSocket
+     * @description CLI-facing binding of {@link CriEndpoint.resolveCriSocket}.
+     * @param {object} [options] - Cluster options (`k3s`, `criSocket`).
+     * @returns {string} CRI endpoint URI.
+     * @memberof UnderpostCluster
+     */
+    resolveCriSocket,
+
+    /**
+     * @method crictlCommandFactory
+     * @description CLI-facing binding of {@link CriEndpoint.crictlCommandFactory}.
+     * @param {string} args - crictl subcommand and arguments (e.g. `pull mongo:latest`).
+     * @param {object} [options] - Cluster options (`k3s`, `criSocket`).
+     * @returns {string} Full shell command.
+     * @memberof UnderpostCluster
+     */
+    crictlCommandFactory,
+
     /**
      * @method config
      * @description Configures host-level settings required for Kubernetes.
@@ -889,7 +1568,7 @@ net.ipv4.ip_forward = 1' | sudo tee /etc/sysctl.d/99-k3s.conf > /dev/null`,
      * @param {object} [options]
      * @param {boolean} [options.all=true] - Remove all unused images, not just dangling ones.
      * @param {boolean} [options.crictl=false] - Also prune the CRI runtime via crictl.
-     * @param {string} [options.criSocket] - Optional crictl --runtime-endpoint override.
+     * @param {string} [options.criSocket] - Optional CRI endpoint override; otherwise the live runtime is resolved.
      * @private
      */
     _pruneContainerCaches(options = {}) {
@@ -906,11 +1585,9 @@ net.ipv4.ip_forward = 1' | sudo tee /etc/sysctl.d/99-k3s.conf > /dev/null`,
         silentOnError: true,
       });
       if (options.crictl) {
-        const ep = options.criSocket ? `--runtime-endpoint ${options.criSocket} ` : '';
-        shellExec(
-          `if command -v crictl >/dev/null 2>&1; then sudo env PATH="$PATH:/usr/local/bin:/usr/bin" crictl ${ep}rmi --prune; fi`,
-          { silentOnError: true },
-        );
+        shellExec(`if command -v crictl >/dev/null 2>&1; then ${crictlCommandFactory('rmi --prune', options)}; fi`, {
+          silentOnError: true,
+        });
       }
       Underpost.cluster._unmountOrphanContainerOverlays();
     },
@@ -1039,11 +1716,15 @@ fi`);
       Underpost.cluster._lazyUmountKubeletMounts();
 
       logger.info('Phase 4/7: Killing control-plane processes and running kubeadm reset...');
-      shellExec(`if command -v crictl >/dev/null 2>&1; then sudo crictl rm -a -f; fi`, { silentOnError: true });
+      shellExec(`if command -v crictl >/dev/null 2>&1; then ${crictlCommandFactory('rm -a -f')}; fi`, {
+        silentOnError: true,
+      });
       // Remove CNI config before stopping sandboxes so Calico's CNI delete hook is
       // not invoked (the API server is already down and the hook would fail).
       shellExec(`sudo rm -rf /etc/cni/net.d/*`);
-      shellExec(`if command -v crictl >/dev/null 2>&1; then sudo crictl rmp -a -f; fi`, { silentOnError: true });
+      shellExec(`if command -v crictl >/dev/null 2>&1; then ${crictlCommandFactory('rmp -a -f')}; fi`, {
+        silentOnError: true,
+      });
       shellExec(`if systemctl is-active --quiet etcd; then sudo systemctl stop etcd; fi`);
       for (const port of [6443, 10259, 10257, 2379, 2380]) {
         shellExec(`if sudo fuser ${port}/tcp >/dev/null 2>&1; then sudo fuser -k ${port}/tcp; fi`);
@@ -1226,6 +1907,10 @@ EOF`);
         shellExec(`sudo rm -rf get_helm.sh`);
       }
 
+      // SOPS and Age for Git-native encrypted secret manifests. Owned by UnderpostSecret so the
+      // same idempotent install backs `underpost secret --install-tools`.
+      Underpost.secret.sops.installTooling();
+
       // Install snap
       shellExec(`sudo yum install -y snapd`);
       shellExec(`sudo systemctl enable --now snapd.socket`);