underpost 3.2.90 → 3.3.0

package/src/server/underpost-compression.js

@@ -0,0 +1,186 @@
+/**
+ * Response compression policy for the Nginx workloads on the request path.
+ *
+ * @module src/server/underpost-compression.js
+ * @namespace UnderpostCompression
+ */
+
+/**
+ * @constant UNDERPOST_COMPRESSION
+ * @description The compression policy both edge workloads render from.
+ *
+ * `text/html` appears in no type list on purpose: nginx and ngx_brotli both
+ * always compress it, and naming it again only invites the two lists to drift.
+ * Every entry is a format that is still text on the wire — already-compressed
+ * media (png, woff2, mp4) is excluded, since re-encoding it spends CPU to add
+ * bytes.
+ * @memberof UnderpostCompression
+ */
+const UNDERPOST_COMPRESSION = {
+  types: [
+    'application/atom+xml',
+    'application/geo+json',
+    'application/javascript',
+    'application/json',
+    'application/ld+json',
+    'application/manifest+json',
+    'application/rss+xml',
+    'application/vnd.api+json',
+    'application/wasm',
+    'application/x-javascript',
+    'application/xhtml+xml',
+    'application/xml',
+    'font/otf',
+    'font/ttf',
+    'image/svg+xml',
+    'image/x-icon',
+    'text/css',
+    'text/javascript',
+    'text/markdown',
+    'text/plain',
+    'text/xml',
+  ],
+  // Below roughly this size a compressed body plus its headers is no smaller
+  // than the original, and can be larger.
+  minLength: 512,
+  // Level 5 of 9: within a few percent of maximum ratio at a fraction of the
+  // CPU. The edge compresses every response of every host, so the cost of a
+  // higher level is paid on the shared workload, not on one deploy.
+  gzipLevel: 5,
+  // Brotli's scale runs to 11, where dynamic compression becomes slower than
+  // the transfer it saves. 5 is the usual on-the-fly ceiling.
+  brotliLevel: 5,
+  filterModule: 'ngx_http_brotli_filter_module.so',
+  staticModule: 'ngx_http_brotli_static_module.so',
+  defaultImage: 'nginx:alpine',
+  env: {
+    image: 'UNDERPOST_NGINX_IMAGE',
+    brotliModules: 'UNDERPOST_NGINX_BROTLI_MODULES',
+    enabled: 'UNDERPOST_NGINX_COMPRESSION',
+  },
+};
+
+/**
+ * @method nginxImageFactory
+ * @description The image both edge workloads run.
+ *
+ * Resolved in one place because the two have to agree: they sit on the same
+ * request path, and brotli is a property of the image rather than of the
+ * config. An image carrying ngx_brotli is the only way the brotli directives
+ * below can be rendered at all.
+ * @returns {string} Container image reference.
+ * @memberof UnderpostCompression
+ */
+const nginxImageFactory = () =>
+  `${process.env[UNDERPOST_COMPRESSION.env.image] || ''}`.trim() || UNDERPOST_COMPRESSION.defaultImage;
+
+/**
+ * @method brotliModuleDirFactory
+ * @description Directory holding the brotli dynamic modules, or an empty string
+ * when the image is not declared to carry them.
+ *
+ * Empty is the default and the safe state: gzip alone, on a stock image, with
+ * no `load_module` line that could fail to resolve.
+ * @returns {string} Absolute directory path, or `''`.
+ * @memberof UnderpostCompression
+ */
+const brotliModuleDirFactory = () =>
+  `${process.env[UNDERPOST_COMPRESSION.env.brotliModules] || ''}`.trim().replace(/\/+$/, '');
+
+/**
+ * @method compressionEnabledFactory
+ * @description Whether responses are compressed at all.
+ *
+ * The escape hatch for a CPU-bound edge node: compression trades cycles for
+ * bytes, and an operator who is out of the former rather than the latter needs
+ * to turn it off without editing generated configuration.
+ * @returns {boolean} True unless explicitly disabled.
+ * @memberof UnderpostCompression
+ */
+const compressionEnabledFactory = () =>
+  !['off', '0', 'false', 'no'].includes(`${process.env[UNDERPOST_COMPRESSION.env.enabled] || ''}`.trim().toLowerCase());
+
+/**
+ * @method compressionModulesConfFactory
+ * @description The `load_module` lines brotli needs, in nginx's main context.
+ *
+ * Rendered only alongside the brotli directives that require them, so a config
+ * never loads a module it does not use or uses one it did not load — using one
+ * that was not loaded is a start-up failure, and the reverse is surface on a
+ * workload that has no documents to serve from it.
+ * @param {string} [brotliModuleDir] - Directory holding the modules; empty renders nothing.
+ * @param {boolean} [enabled] - Whether compression is on at all.
+ * @param {boolean} [staticRoot] - Whether the workload serves documents from disk.
+ * @returns {string} Main-context directives, or an empty string.
+ * @memberof UnderpostCompression
+ */
+const compressionModulesConfFactory = ({
+  brotliModuleDir = brotliModuleDirFactory(),
+  enabled = compressionEnabledFactory(),
+  staticRoot = false,
+} = {}) => {
+  if (!enabled || !brotliModuleDir) return '';
+  return [
+    `load_module ${brotliModuleDir}/${UNDERPOST_COMPRESSION.filterModule};`,
+    ...(staticRoot ? [`load_module ${brotliModuleDir}/${UNDERPOST_COMPRESSION.staticModule};`] : []),
+  ].join('\n');
+};
+
+/**
+ * @method compressionConfFactory
+ * @description The compression block for an nginx `http` context.
+ *
+ * `gzip_proxied any` is what makes this worth rendering on a reverse proxy at
+ * all: the default is `off`, which suppresses compression for exactly the
+ * responses these workloads exist to forward. A response the upstream already
+ * encoded is passed through untouched either way — nginx never re-compresses
+ * one that carries `Content-Encoding`.
+ *
+ * `*_static` is only rendered where a document root exists. It costs one stat()
+ * per request to serve an operator-placed `.gz`/`.br` sibling with no CPU at
+ * all, and has nothing to look for on a workload that serves no files.
+ * @param {boolean} [enabled] - Whether compression is on at all.
+ * @param {string} [brotliModuleDir] - Directory holding the brotli modules; empty renders gzip alone.
+ * @param {boolean} [staticRoot] - Whether the workload serves documents from disk.
+ * @param {string} [indent] - Leading whitespace for each line.
+ * @returns {string} `http`-context directives, or an empty string.
+ * @memberof UnderpostCompression
+ */
+const compressionConfFactory = ({
+  enabled = compressionEnabledFactory(),
+  brotliModuleDir = brotliModuleDirFactory(),
+  staticRoot = false,
+  indent = '  ',
+} = {}) => {
+  if (!enabled) return '';
+  const types = UNDERPOST_COMPRESSION.types.join(' ');
+  const lines = [
+    'gzip on;',
+    // Compressed and uncompressed bodies share a URL, so a cache that ignores
+    // this header serves one to a client that asked for the other.
+    'gzip_vary on;',
+    `gzip_comp_level ${UNDERPOST_COMPRESSION.gzipLevel};`,
+    `gzip_min_length ${UNDERPOST_COMPRESSION.minLength};`,
+    'gzip_proxied any;',
+    `gzip_types ${types};`,
+    ...(staticRoot ? ['gzip_static on;'] : []),
+  ];
+  if (brotliModuleDir)
+    lines.push(
+      'brotli on;',
+      `brotli_comp_level ${UNDERPOST_COMPRESSION.brotliLevel};`,
+      `brotli_min_length ${UNDERPOST_COMPRESSION.minLength};`,
+      `brotli_types ${types};`,
+      ...(staticRoot ? ['brotli_static on;'] : []),
+    );
+  return lines.map((line) => `${indent}${line}`).join('\n');
+};
+
+export {
+  UNDERPOST_COMPRESSION,
+  brotliModuleDirFactory,
+  compressionConfFactory,
+  compressionEnabledFactory,
+  compressionModulesConfFactory,
+  nginxImageFactory,
+};

package/src/server/underpost-gateway.js

@@ -32,6 +32,7 @@ import fs from 'fs-extra';
 import nodePath from 'node:path';
 import { timer } from '../client/components/core/CommonJs.js';
 import { instanceStatusPageEntriesFactory, loadConfServerJson, loadReplicas } from './conf.js';
+import { compressionConfFactory, compressionModulesConfFactory, nginxImageFactory } from './underpost-compression.js';
 import Underpost from '../index.js';
 import { loggerFactory } from './logger.js';
 import { shellExec } from './process.js';
@@ -51,7 +52,7 @@ const UNDERPOST_GATEWAY = {
   configMapName: 'underpost-gateway-nginx',
   claimName: 'pvc-underpost-gateway',
   volumeName: 'pv-underpost-gateway',
-  image: 'nginx:alpine',
+  image: nginxImageFactory(),
   root: '/var/www/static',
   port: 80,
   healthPath: '/healthz',
@@ -189,13 +190,24 @@ const defaultStatusPagePath = (status = 404) => `${UNDERPOST_GATEWAY.defaultHost
  * 200. That status is what stops the document being stored: the PWA service
  * worker caches navigations for hours and would otherwise keep serving the
  * fallback long after the host's real page landed in the tree.
+ *
+ * Compression is applied here rather than further out because this is the last
+ * hop that holds a response body in the clear — and, for every host whose
+ * errors are intercepted, the hop every site path already passes through. See
+ * {@link module:src/server/underpost-compression.js}.
+ * @param {string} [resolver] - Cluster DNS ClusterIP.
+ * @param {object} [compression] - Overrides for the compression policy.
  * @returns {string} nginx.conf contents.
  * @memberof UnderpostGateway
  */
-const nginxConfFactory = ({ resolver = UNDERPOST_GATEWAY.resolver } = {}) => `worker_processes auto;
+const nginxConfFactory = ({ resolver = UNDERPOST_GATEWAY.resolver, compression = {} } = {}) => {
+  const policy = { ...compression, staticRoot: true };
+  const modules = compressionModulesConfFactory(policy);
+  const compress = compressionConfFactory(policy);
+  return `worker_processes auto;
 error_log /dev/stderr warn;
 pid /tmp/nginx.pid;
-
+${modules ? `${modules}\n` : ''}
 events {
   worker_connections 1024;
 }
@@ -206,12 +218,7 @@ http {
   sendfile on;
   tcp_nopush on;
   server_tokens off;
-
-  gzip on;
-  gzip_vary on;
-  gzip_min_length 512;
-  gzip_types text/html text/css text/plain application/javascript application/json image/svg+xml;
-
+${compress ? `\n${compress}\n` : ''}
   log_format concise '$remote_addr "$request" $status $body_bytes_sent "$host"';
   access_log /dev/stdout concise;
 
@@ -259,6 +266,7 @@ http {
   }
 }
 `;
+};
 
 /**
  * @method statusPageLocationsFactory
@@ -388,6 +396,7 @@ ${blocks
  * @param {string} hostPath - Node directory backing the static root.
  * @param {string} [nodeName] - Node the hostPath volume is pinned to.
  * @param {string} [storage] - Volume size.
+ * @param {object} [compression] - Overrides for the compression policy.
  * @returns {string} Multi-document YAML.
  * @memberof UnderpostGateway
  */
@@ -397,8 +406,9 @@ const underpostGatewayManifestsFactory = ({
   nodeName = '',
   storage = '1Gi',
   resolver,
+  compression = {},
 } = {}) => {
-  const nginxConf = nginxConfFactory({ resolver });
+  const nginxConf = nginxConfFactory({ resolver, compression });
   // The config is mounted with `subPath`, which Kubernetes never refreshes in
   // place, and the pod template is otherwise identical across rebuilds — so
   // without this annotation an edited nginx.conf reaches the ConfigMap and

package/src/server/underpost-ingress.js

@@ -24,6 +24,8 @@
  * @namespace UnderpostIngress
  */
 
+import { compressionConfFactory, compressionModulesConfFactory, nginxImageFactory } from './underpost-compression.js';
+
 /**
  * @constant UNDERPOST_INGRESS
  * @description Identity of the underpost ingress workload. One deployment fronts every
@@ -33,7 +35,7 @@
 const UNDERPOST_INGRESS = {
   name: 'underpost-ingress',
   configMapName: 'underpost-ingress-nginx',
-  image: 'nginx:alpine',
+  image: nginxImageFactory(),
   httpPort: 80,
   httpsPort: 443,
   healthPort: 8090,
@@ -100,10 +102,17 @@ const underpostIngressHostMapFactory = ({ contourHosts = [], gatewayHosts = [],
  * that owns most of the routing rather than a rejection, so a host whose route
  * object has not been applied yet still reaches a data plane that can answer it
  * — including with its own 404.
+ *
+ * Compression is rendered for `:80` alone, and that asymmetry is the same split
+ * the module makes everywhere else: an L7 hop holds a body it can encode, while
+ * `:443` carries bytes this process never decrypts. Whatever leaves through the
+ * tunnel on that port was compressed where TLS terminates — see
+ * {@link module:src/server/underpost-compression.js}.
  * @param {Array<object>} [entries] - Host table from {@link UnderpostIngress.underpostIngressHostMapFactory}.
  * @param {object} backends - `{contour: {http, tls}, gateway: {http, tls}}`; a missing stack is omitted.
  * @param {string} [defaultBackend] - Backend for an unmatched hostname.
  * @param {string} [resolver] - Cluster DNS ClusterIP.
+ * @param {object} [compression] - Overrides for the compression policy.
  * @returns {string} nginx.conf contents.
  * @memberof UnderpostIngress
  */
@@ -112,6 +121,7 @@ const underpostIngressConfFactory = ({
   backends = {},
   defaultBackend = 'gateway',
   resolver = UNDERPOST_INGRESS.resolver,
+  compression = {},
 } = {}) => {
   const available = Object.keys(backends).filter((name) => backends[name]?.http && backends[name]?.tls);
   // With one stack installed the map still renders, so the same workload serves
@@ -121,17 +131,23 @@ const underpostIngressConfFactory = ({
   const routable = entries.filter((entry) => entry?.host && available.includes(entry.backend));
   const mapEntries = (kind) =>
     routable.map((entry) => `    ${entry.host} ${backends[entry.backend][kind]};`).join('\n') || '';
+  // No `staticRoot`: this workload proxies every byte and serves no documents,
+  // so there is never a pre-compressed sibling on disk to prefer.
+  const policy = { ...compression, staticRoot: false };
+  const modules = compressionModulesConfFactory(policy);
+  const compress = compressionConfFactory(policy);
 
   return `worker_processes auto;
 error_log /dev/stderr warn;
 pid /tmp/nginx.pid;
-
+${modules ? `${modules}\n` : ''}
 events {
   worker_connections 4096;
 }
 
 http {
   server_tokens off;
+${compress ? `\n${compress}\n` : ''}
   log_format underpost_ingress '$remote_addr "$request" $status "$host" -> $underpost_ingress_http_upstream';
   access_log /dev/stdout underpost_ingress;
 

package/test/selinux.test.js

@@ -0,0 +1,71 @@
+'use strict';
+
+import { expect } from 'chai';
+import SELinuxService, {
+  runSELinuxCommands,
+  selinuxContainerSharedContextCommandsFactory,
+  selinuxEnforcingCommandsFactory,
+  selinuxFileContextCommandFactory,
+  selinuxRestoreconCommandFactory,
+  selinuxSshContextCommandsFactory,
+  selinuxSshPortCommandsFactory,
+  shellArgumentFactory,
+} from '../src/server/selinux.js';
+
+describe('SELinux utilities', () => {
+  it('provides one canonical class API with named factory aliases', () => {
+    expect(SELinuxService.selinuxEnforcingCommandsFactory).to.equal(selinuxEnforcingCommandsFactory);
+    expect(SELinuxService.runSELinuxCommands).to.equal(runSELinuxCommands);
+    expect(SELinuxService.shellArgumentFactory).to.equal(shellArgumentFactory);
+  });
+
+  it('quotes shell arguments and restores policy contexts', () => {
+    expect(shellArgumentFactory("/home/o'hara/.ssh")).to.equal("'/home/o'\"'\"'hara/.ssh'");
+    const command = selinuxRestoreconCommandFactory(['/root/.ssh', '/etc/sudoers.d/90_admin']);
+    expect(command).to.include("restorecon -RF '/root/.ssh'");
+    expect(command).to.include("restorecon -RF '/etc/sudoers.d/90_admin'");
+  });
+
+  it('shares container-mounted host paths with a persistent container_file_t mapping', () => {
+    const commands = selinuxContainerSharedContextCommandsFactory(['/etc/kubernetes', '/var/lib/etcd']);
+    expect(commands).to.have.length(3);
+    expect(commands[0]).to.include("semanage fcontext -a -t container_file_t '/etc/kubernetes(/.*)?'");
+    expect(commands[0]).to.include("semanage fcontext -m -t container_file_t '/etc/kubernetes(/.*)?'");
+    expect(commands[1]).to.include("container_file_t '/var/lib/etcd(/.*)?'");
+    expect(commands[2]).to.include("restorecon -RF '/etc/kubernetes'");
+    expect(() => selinuxContainerSharedContextCommandsFactory([])).to.throw(TypeError);
+  });
+
+  it('requires both a path and a type for file context mappings', () => {
+    expect(() => selinuxFileContextCommandFactory('', { type: 'container_file_t' })).to.throw(TypeError);
+    expect(() => selinuxFileContextCommandFactory('/var/lib/etcd', {})).to.throw(TypeError);
+  });
+
+  it('adds persistent mappings only for non-standard SSH homes', () => {
+    expect(selinuxSshContextCommandsFactory({ sshDirectory: '/home/admin/.ssh' })).to.have.length(1);
+    expect(selinuxSshContextCommandsFactory({ sshDirectory: '/srv/admin/.ssh' })[0]).to.include(
+      'semanage fcontext -a -t ssh_home_t',
+    );
+  });
+
+  it('restores requested contexts before activating Enforcing mode', () => {
+    const commands = selinuxEnforcingCommandsFactory({ restorePaths: ['/var/lib/containerd'] });
+    expect(commands[1]).to.include('touch /.autorelabel');
+    expect(commands[2]).to.include('restorecon');
+    expect(commands[3]).to.include('setenforce 1');
+  });
+
+  it('maps only custom SSH ports and validates the port range', () => {
+    expect(selinuxSshPortCommandsFactory({ port: 22 })).to.deep.equal([]);
+    expect(selinuxSshPortCommandsFactory({ port: 2222 })[0]).to.include(
+      'semanage port -a -t ssh_port_t -p tcp 2222',
+    );
+    expect(() => selinuxSshPortCommandsFactory({ port: 70000 })).to.throw(RangeError);
+  });
+
+  it('executes generated commands with an injected executor', () => {
+    const executed = [];
+    runSELinuxCommands(['one', 'two'], { execute: (command) => executed.push(command) });
+    expect(executed).to.deep.equal(['one', 'two']);
+  });
+});

package/test/underpost-gateway.test.js

@@ -119,6 +119,47 @@ describe('underpost gateway edge tier', () => {
       expect(conf).to.include('map $http_upgrade $connection_upgrade');
     });
 
+    // For every host whose errors are intercepted, this workload is the hop the
+    // site path already passes through — and the last one holding the body in
+    // the clear. Compressing anywhere further out is compressing ciphertext.
+    describe('response compression', () => {
+      it('compresses what it serves and what it proxies', () => {
+        expect(conf).to.include('gzip on;');
+        expect(conf).to.include('gzip_vary on;');
+        // Nginx's default is `off`, which skips the proxied responses that carry
+        // most of the bytes this workload forwards.
+        expect(conf).to.include('gzip_proxied any;');
+        expect(conf).to.match(/gzip_types [^;]*application\/javascript[^;]*;/);
+        expect(conf).to.match(/gzip_types [^;]*text\/css[^;]*;/);
+      });
+
+      // Documents are served from a volume, so an operator who places a `.gz`
+      // beside one gets it sent with no compression pass at all.
+      it('prefers a pre-compressed sibling from the static root', () => {
+        expect(conf).to.include('gzip_static on;');
+      });
+
+      it('renders brotli and its modules only when the image is declared to carry them', () => {
+        expect(conf).to.not.include('brotli');
+        expect(conf).to.not.include('load_module');
+        const declared = nginxConfFactory({ compression: { brotliModuleDir: '/usr/lib/nginx/modules' } });
+        expect(declared).to.include('load_module /usr/lib/nginx/modules/ngx_http_brotli_static_module.so;');
+        expect(declared).to.include('brotli on;');
+        expect(declared).to.include('brotli_static on;');
+        // Main context: nginx rejects `load_module` anywhere else.
+        expect(declared.indexOf('load_module')).to.be.lessThan(declared.indexOf('events {'));
+      });
+
+      // The config is mounted with `subPath`, which Kubernetes never refreshes,
+      // so only a changed pod template rolls the workload onto a new policy.
+      it('rolls the deployment when the policy changes', () => {
+        const hash = (docs) => kind(docs, 'Deployment').match(/nginx-conf-hash: '([0-9a-f]+)'/)[1];
+        expect(hash(manifests({ compression: { brotliModuleDir: '/usr/lib/nginx/modules' } }))).to.not.equal(
+          hash(manifests()),
+        );
+      });
+    });
+
     // A 200 would let the PWA service worker store the shared page as the
     // host's own and keep serving it after the real document lands.
     it('serves the shared fallback as a 404 that is never stored', () => {

package/test/underpost-ingress.test.js

@@ -123,6 +123,58 @@ describe('shared ingress front', () => {
       const out = conf({ entries: [], resolver: '10.43.0.10' });
       expect(out.match(/resolver 10\.43\.0\.10 valid=10s ipv6=off;/g)).to.have.lengthOf(2);
     });
+
+    describe('response compression', () => {
+      const halves = (input) => {
+        const out = conf({ entries: [], ...input });
+        const split = out.indexOf('\nstream {');
+        return { http: out.slice(0, split), stream: out.slice(split) };
+      };
+
+      it('compresses the cleartext hop it proxies at L7', () => {
+        const { http } = halves({ compression: { brotliModuleDir: '' } });
+        expect(http).to.include('gzip on;');
+        expect(http).to.include('gzip_proxied any;');
+        expect(http).to.include('gzip_vary on;');
+      });
+
+      // `:443` is forwarded by SNI without being decrypted, so there is no body
+      // in this context to encode — and no compression directive is even valid
+      // in `stream`.
+      it('leaves the L4 contexts alone', () => {
+        const { stream } = halves({ compression: { brotliModuleDir: '/usr/lib/nginx/modules' } });
+        expect(stream).to.not.include('gzip');
+        expect(stream).to.not.include('brotli');
+      });
+
+      // This workload proxies every byte and serves no documents, so there is
+      // never a pre-compressed sibling on disk to look for.
+      it('does not look for pre-compressed files it can never have', () => {
+        const { http } = halves({ compression: { brotliModuleDir: '/usr/lib/nginx/modules' } });
+        expect(http).to.not.include('_static');
+      });
+
+      // `brotli on;` in an image without the module is an unknown directive, and
+      // this workload holds the node's 80/443 — a start-up failure here is the
+      // whole edge, so brotli is rendered only where it is declared to exist.
+      it('renders brotli and its modules only when the image is declared to carry them', () => {
+        const stock = conf({ entries: [], compression: { brotliModuleDir: '' } });
+        expect(stock).to.not.include('brotli');
+        expect(stock).to.not.include('load_module');
+        const declared = conf({ entries: [], compression: { brotliModuleDir: '/usr/lib/nginx/modules' } });
+        expect(declared).to.include('load_module /usr/lib/nginx/modules/ngx_http_brotli_filter_module.so;');
+        expect(declared).to.include('brotli on;');
+        // Main context: nginx rejects `load_module` anywhere else.
+        expect(declared.indexOf('load_module')).to.be.lessThan(declared.indexOf('events {'));
+      });
+
+      it('renders no compression directive when compression is switched off', () => {
+        const out = conf({ entries: [], compression: { enabled: false, brotliModuleDir: '/usr/lib/nginx/modules' } });
+        expect(out).to.not.include('gzip');
+        expect(out).to.not.include('brotli');
+        expect(out).to.not.include('load_module');
+      });
+    });
   });
 
   describe('underpostIngressManifestsFactory', () => {