underpost 3.2.21 → 3.2.28

package/src/runtime/nginx/Nginx.js

@@ -0,0 +1,250 @@
+/**
+ * Exported singleton instance of the NginxService class.
+ * Manages dynamic generation of nginx reverse-proxy router configuration used
+ * by the Docker Compose development stack. Mirrors the conventions of
+ * {@link module:src/runtime/lampp/Lampp.js LamppService}.
+ * @module src/runtime/nginx/Nginx.js
+ * @namespace NginxService
+ */
+
+import fs from 'fs-extra';
+import path from 'path';
+import { loggerFactory } from '../../server/logger.js';
+
+const logger = loggerFactory(import.meta);
+
+/**
+ * @class NginxService
+ * @description Builds nginx `server` blocks (the router) for fronting upstream
+ * application services and writes the rendered configuration to disk. The
+ * router is accumulated in memory via {@link NginxService#createApp} and
+ * flushed with {@link NginxService#writeConf}, keeping config generation
+ * decoupled from the filesystem location it is written to.
+ * @memberof NginxService
+ */
+class NginxService {
+  /**
+   * @type {string}
+   * @description Accumulated nginx `server { ... }` blocks (the router definition).
+   * @memberof NginxService
+   */
+  router = '';
+
+  /**
+   * @type {Set<string>}
+   * @description Upstream blocks keyed by upstream name to avoid duplicates.
+   * @memberof NginxService
+   */
+  upstreams;
+
+  /**
+   * @type {boolean}
+   * @description Whether a default_server catch-all block has been emitted.
+   * @memberof NginxService
+   */
+  hasDefaultServer;
+
+  constructor() {
+    this.reset();
+  }
+
+  /**
+   * Resets the in-memory router, upstreams, and default-server flag.
+   * @method reset
+   * @returns {void}
+   * @memberof NginxService
+   */
+  reset() {
+    this.router = '';
+    this.upstreams = new Map();
+    this.hasDefaultServer = false;
+  }
+
+  /**
+   * Appends a raw render fragment to the router string.
+   * @method appendRouter
+   * @param {string} render - The configuration fragment to append.
+   * @returns {string} The complete, updated router configuration string.
+   * @memberof NginxService
+   */
+  appendRouter(render) {
+    if (!this.router) return (this.router = render);
+    return (this.router += render);
+  }
+
+  /**
+   * Clears the in-memory router configuration.
+   * @method removeRouter
+   * @returns {void}
+   * @memberof NginxService
+   */
+  removeRouter() {
+    this.reset();
+  }
+
+  /**
+   * Registers a named upstream pointing at a container service:port.
+   * Idempotent: repeated names with the same target are collapsed.
+   * @method addUpstream
+   * @param {string} name - The upstream identifier.
+   * @param {string} service - The Docker service name (resolved via service discovery).
+   * @param {number} port - The upstream container port.
+   * @returns {string} The upstream name.
+   * @memberof NginxService
+   */
+  addUpstream(name, service, port) {
+    this.upstreams.set(name, `upstream ${name} { server ${service}:${port}; }`);
+    return name;
+  }
+
+  /**
+   * Renders the standard proxy directive block shared by all locations,
+   * including websocket upgrade headers and forwarded headers.
+   * @method proxyLocation
+   * @param {string} location - The location path (e.g. '/', '/peer').
+   * @param {string} upstream - The upstream name to proxy to.
+   * @returns {string} The rendered `location { ... }` block.
+   * @memberof NginxService
+   */
+  proxyLocation(location, upstream) {
+    return `
+    location ${location} {
+        proxy_pass http://${upstream};
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection $connection_upgrade;
+        proxy_read_timeout 3600s;
+    }
+`;
+  }
+
+  /**
+   * Renders the `/healthz` location used by the proxy container healthcheck.
+   * @method healthLocation
+   * @returns {string} The rendered health-check location block.
+   * @memberof NginxService
+   */
+  healthLocation() {
+    return `
+    location = /healthz {
+        access_log off;
+        return 200 "ok\\n";
+        add_header Content-Type text/plain;
+    }
+`;
+  }
+
+  /**
+   * Creates an nginx virtual-host (`server`) entry for a host and appends it to
+   * the router. Each route maps a URL prefix to an upstream service:port with
+   * websocket support, mirroring the Contour HTTPProxy route model.
+   *
+   * @method createApp
+   * @param {object} options - Virtual host options.
+   * @param {string} options.host - The `server_name` (e.g. 'default.net').
+   * @param {number} [options.listen=80] - The listen port.
+   * @param {Array<{location: string, service: string, port: number}>} options.routes
+   *   - Route table. Longer prefixes should precede '/' for correct matching.
+   * @param {boolean} [options.resetRouter] - Clear the router before appending.
+   * @returns {string} The complete, updated router configuration string.
+   * @memberof NginxService
+   */
+  createApp({ host, listen = 80, routes = [], resetRouter = false }) {
+    if (resetRouter) this.removeRouter();
+
+    const safeHost = host.replace(/[^a-zA-Z0-9]/g, '_');
+    const locationBlocks = routes
+      .map(({ location, service, port }) => {
+        const upstreamName = this.addUpstream(`up_${safeHost}_${port}`, service, port);
+        return this.proxyLocation(location, upstreamName);
+      })
+      .join('');
+
+    this.appendRouter(`
+server {
+    listen ${listen};
+    server_name ${host};
+${this.healthLocation()}${locationBlocks}}
+`);
+
+    return this.router;
+  }
+
+  /**
+   * Emits a `default_server` catch-all that forwards to the given upstream and
+   * answers the health probe for unmatched Host headers. Safe to call once.
+   *
+   * @method createDefaultServer
+   * @param {object} options - Default server options.
+   * @param {string} options.service - The Docker service name to fall back to.
+   * @param {number} options.port - The upstream container port.
+   * @param {number} [options.listen=80] - The listen port.
+   * @returns {string} The complete, updated router configuration string.
+   * @memberof NginxService
+   */
+  createDefaultServer({ service, port, listen = 80 }) {
+    if (this.hasDefaultServer) return this.router;
+    this.hasDefaultServer = true;
+    const upstreamName = this.addUpstream('up_default', service, port);
+    this.appendRouter(`
+server {
+    listen ${listen} default_server;
+    server_name _;
+${this.healthLocation()}${this.proxyLocation('/', upstreamName)}}
+`);
+    return this.router;
+  }
+
+  /**
+   * Renders the full nginx config document: the websocket connection map, all
+   * upstream blocks, the global `proxy_http_version`, and the accumulated router.
+   * @method render
+   * @returns {string} The complete nginx configuration file content.
+   * @memberof NginxService
+   */
+  render() {
+    const upstreamBlocks = Array.from(this.upstreams.values()).join('\n');
+    return `# Generated by src/runtime/nginx/Nginx.js — do not hand-edit.
+# Reverse proxy derived from manifests/deployment/*/proxy.yaml (Contour HTTPProxy).
+# Upstreams resolve container services via the Docker internal network.
+
+map $http_upgrade $connection_upgrade {
+    default upgrade;
+    ''      close;
+}
+
+${upstreamBlocks}
+
+proxy_http_version 1.1;
+${this.router}`;
+  }
+
+  /**
+   * Writes the rendered configuration to the given path, creating parent
+   * directories as needed. Idempotent and safe to rerun.
+   * @method writeConf
+   * @param {string} confPath - Absolute or relative destination path.
+   * @returns {string} The path written.
+   * @memberof NginxService
+   */
+  writeConf(confPath) {
+    const target = path.resolve(confPath);
+    fs.mkdirpSync(path.dirname(target));
+    fs.writeFileSync(target, this.render(), 'utf8');
+    logger.info(`nginx config written`, { path: target, upstreams: this.upstreams.size });
+    return target;
+  }
+}
+
+/**
+ * @description Exported singleton instance of the NginxService class.
+ * @type {NginxService}
+ * @memberof NginxService
+ */
+const Nginx = new NginxService();
+
+export { Nginx, NginxService };
+export default Nginx;

package/src/server/catalog.js

@@ -18,8 +18,6 @@ import fs from 'fs-extra';
 import { fileURLToPath } from 'node:url';
 import * as path from 'node:path';
 
-const catalogDir = path.dirname(fileURLToPath(import.meta.url));
-
 /** Empty product catalog returned for deploy ids without a dedicated module. */
 const EMPTY_CATALOG = {
   sourceMoves: [],
@@ -43,12 +41,11 @@ const EMPTY_CATALOG = {
 const loadDeployCatalog = async (deployId) => {
   const suffix = (deployId ?? '').split('dd-')[1];
   if (!suffix) return EMPTY_CATALOG;
-  try {
-    const mod = await import(`./catalog-${suffix}.js`);
+  if (fs.existsSync(`./src/projects/${suffix}/catalog-${suffix}.js`)) {
+    const mod = await import(`../projects/${suffix}/catalog-${suffix}.js`);
     return { ...EMPTY_CATALOG, ...(mod.default ?? {}) };
-  } catch {
-    return EMPTY_CATALOG;
   }
+  return EMPTY_CATALOG;
 };
 
 /**
@@ -62,14 +59,10 @@ const loadDeployCatalog = async (deployId) => {
  */
 const loadProductCatalogs = async () => {
   const catalogs = [];
-  for (const file of fs.readdirSync(catalogDir)) {
-    if (!/^catalog-.+\.js$/.test(file) || file === 'catalog-underpost.js') continue;
-    try {
-      const mod = await import(`./${file}`);
-      if (mod.default) catalogs.push({ ...EMPTY_CATALOG, ...mod.default });
-    } catch {
-      /* a malformed/removed product catalog must not break the base build */
-    }
+  for (const file of await fs.readdir('./src/projects')) {
+    if (file === 'underpost') continue;
+    const mod = await import(`../projects/${file}/catalog-${file}.js`);
+    if (mod.default) catalogs.push({ ...EMPTY_CATALOG, ...mod.default });
   }
   return catalogs;
 };

package/src/server/conf.js

@@ -1337,7 +1337,7 @@ const mergeFile = async (parts = [], outputFilePath) => {
  * @memberof ServerConfBuilder
  */
 const getPathsSSR = (conf) => {
-  const paths = ['src/client/ssr/Render.js'];
+  const paths = ['src/client/ssr/RootDocument.js'];
   for (const o of conf.head) paths.push(`src/client/ssr/head/${o}.js`);
   for (const o of conf.body) paths.push(`src/client/ssr/body/${o}.js`);
   for (const o of Object.keys(conf.mailer)) paths.push(`src/client/ssr/mailer/${conf.mailer[o]}.js`);
@@ -1856,6 +1856,7 @@ const syncPrivateConf = (deployId, extraPaths = []) => {
   if (!fs.existsSync(privateRepoPath)) {
     shellExec(`cd .. && underpost clone ${privateGitUri}`, { silent: true });
   } else {
+    shellExec(`git config --global --add safe.directory '${dir.resolve(privateRepoPath)}'`);
     shellExec(`cd ${privateRepoPath} && git checkout . && git clean -f -d && underpost pull . ${privateGitUri}`, {
       silent: true,
     });
@@ -1927,7 +1928,8 @@ const syncDeployIdSources = (sourceMoves = []) => {
  */
 const buildTemplate = async ({ srcPath = './', toPath = '../pwa-microservices-template' } = {}) => {
   const walk = (await import('ignore-walk')).default;
-  const { TEMPLATE_RESTORE_PATHS, TEMPLATE_KEYWORDS, TEMPLATE_DESCRIPTION } = await import('./catalog-underpost.js');
+  const { TEMPLATE_RESTORE_PATHS, TEMPLATE_KEYWORDS, TEMPLATE_DESCRIPTION } =
+    await import('../projects/underpost/catalog-underpost.js');
   const { loadProductCatalogs } = await import('./catalog.js');
   const githubUsername = process.env.GITHUB_USERNAME;
 
@@ -1941,15 +1943,9 @@ const buildTemplate = async ({ srcPath = './', toPath = '../pwa-microservices-te
     )
   ).filter((p) => !p.startsWith('.git'));
 
-  // Clone the template from 0 if missing; otherwise reset it to a clean pristine checkout.
-  if (!fs.existsSync(toPath)) {
-    shellExec(`cd .. && node engine/bin clone ${githubUsername}/pwa-microservices-template`);
-  } else {
-    shellExec(`cd ${toPath} && git reset && git checkout . && git clean -f -d`);
-    shellExec(`node bin pull ${toPath} ${githubUsername}/pwa-microservices-template`);
-    shellExec(`sudo rm -rf ${toPath}/engine-private`);
-    shellExec(`sudo rm -rf ${toPath}/logs`);
-  }
+  fs.removeSync(`${githubUsername}/pwa-microservices-template`);
+  shellExec(`cd .. && node engine/bin clone ${githubUsername}/pwa-microservices-template`);
+
   shellExec(`cd ${toPath} && git config core.filemode false`);
 
   for (const copyPath of sourceFiles) {
@@ -1979,7 +1975,6 @@ const buildTemplate = async ({ srcPath = './', toPath = '../pwa-microservices-te
   }
   shellExec(`rm -rf ${toPath}/.github`);
   shellExec(`rm -rf ${toPath}/manifests/deployment/dd-*`);
-  shellExec(`rm -rf ${toPath}/src/server/catalog-*`);
 
   fs.mkdirSync(`${toPath}/.github/workflows`, { recursive: true });
   for (const restorePath of TEMPLATE_RESTORE_PATHS) {
@@ -2052,6 +2047,59 @@ git add .`);
   }
 };
 
+/**
+ * @method updatePrivateEngineTestRepo
+ * @description Publishes a deploy id's freshly assembled template to its private
+ * **test** source repo `engine-test-<idPart>` (separate from the production
+ * `engine-<idPart>`). A pod started with `underpost start --build --private-test-repo`
+ * clones this repo, so work-in-progress engine source can be tested end to end
+ * without touching the production source. Mirrors {@link updatePrivateTemplateRepo}
+ * but per-deploy-id and against the test repo.
+ *
+ * Assumes the deploy id template has already been assembled at the template path
+ * (run `node bin/build <deployId>` first, or use `node bin/build <deployId> --update-private`).
+ * @param {string} deployId - Concrete deploy id (e.g. `dd-core`).
+ * @returns {Promise<void>}
+ * @memberof ServerConfBuilder
+ */
+const updatePrivateEngineTestRepo = async (deployId) => {
+  const username = process.env.GITHUB_USERNAME || 'underpostnet';
+  const repoName = `engine-test-${deployId.split('-')[1]}`;
+  const templatePath = '/home/dd/pwa-microservices-template';
+  if (!fs.existsSync(templatePath))
+    throw new Error(`updatePrivateEngineTestRepo: assemble the template first (node bin/build ${deployId})`);
+
+  // Detach the assembled working tree from any engine-build git history.
+  shellExec(`sudo rm -rf ${templatePath}/.git`);
+
+  // Adopt the test repo's existing history when present (so the push is a delta);
+  // otherwise publish a fresh history on first push.
+  shellExec(`cd /home/dd && sudo rm -rf ./${repoName}.git && underpost clone --bare ${username}/${repoName}`, {
+    silent: true,
+    disableLog: true,
+    silentOnError: true,
+  });
+  if (fs.existsSync(`/home/dd/${repoName}.git`)) shellExec(`mv /home/dd/${repoName}.git ${templatePath}/.git`);
+
+  // `git init` converts the moved bare repo into a normal work-tree repo (bare
+  // clones have no work tree, so `git add` would fail), and bootstraps a fresh
+  // repo on first publish. Idempotent — mirrors updatePrivateTemplateRepo.
+  shellExec(`cd ${templatePath}
+git init
+git config user.name '${username}'
+git config user.email 'development@underpost.net'
+git add .`);
+
+  const hasChanges = shellExec(`node bin cmt ${templatePath} --has-changes`, {
+    stdout: true,
+    silent: true,
+    disableLog: true,
+  }).trim();
+  if (hasChanges === '1')
+    shellExec(`cd ${templatePath} && git commit -m 'Update ${repoName}' && underpost push . ${username}/${repoName}`);
+  else logger.info('No changes to publish', { repoName });
+};
+
 export {
   Config,
   loadConf,
@@ -2104,4 +2152,5 @@ export {
   syncDeployIdSources,
   buildTemplate,
   updatePrivateTemplateRepo,
+  updatePrivateEngineTestRepo,
 };

package/src/server/start.js

@@ -152,10 +152,16 @@ class UnderpostStartUp {
       // observable through every lifecycle phase, including build and init. Bind
       // the deployment-resolved port so it always matches the monitor's target.
       startInternalStatusServer(deployStatusPort(deployId, env));
-      setRuntimeStatus(deployId, env, RUNTIME_STATUS.BUILD);
-      if (options.build === true) await Underpost.start.build(deployId, env, options);
-      setRuntimeStatus(deployId, env, RUNTIME_STATUS.INIT);
-      if (options.run === true) await Underpost.start.run(deployId, env, options);
+      try {
+        setRuntimeStatus(deployId, env, RUNTIME_STATUS.BUILD);
+        if (options.build === true) await Underpost.start.build(deployId, env, options);
+        setRuntimeStatus(deployId, env, RUNTIME_STATUS.INIT);
+        if (options.run === true) await Underpost.start.run(deployId, env, options);
+      } catch (error) {
+        logger.error('Deployment build/init failed', { deployId, env, message: error?.message });
+        setRuntimeStatus(deployId, env, RUNTIME_STATUS.ERROR);
+        if (!Underpost.env.isInsideContainer()) throw error;
+      }
     },
     /**
      * Run itc-scripts and builds client bundle.
@@ -167,6 +173,8 @@ class UnderpostStartUp {
      * @param {boolean} options.skipFullBuild - Whether to skip building the full client bundle.
      * @param {boolean} options.pullBundle - When true, download pre-built client bundle from Cloudinary via pull-bundle (must be pushed first with push-bundle).
      *   This flag is independent of skipFullBuild: it can be combined with skipFullBuild or used alone.
+     * @param {boolean} options.privateTestRepo - When true, clone `engine-test-<id>` (the private test source repo
+     *   published by `node bin/build <deployId> --update-private`) instead of the production `engine-<id>` repo.
      * @memberof UnderpostStartUp
      */
     async build(
@@ -175,7 +183,11 @@ class UnderpostStartUp {
       options = { underpostQuicklyInstall: false, skipPullBase: false, skipFullBuild: false, pullBundle: false },
     ) {
       const buildBasePath = `/home/dd`;
-      const repoName = `engine-${deployId.split('-')[1]}`;
+      // `--private-test-repo` clones the isolated test source repo published by
+      // `node bin/build <deployId> --update-private`, instead of the production one.
+      const repoName = options?.privateTestRepo
+        ? `engine-test-${deployId.split('-')[1]}`
+        : `engine-${deployId.split('-')[1]}`;
       if (!options.skipPullBase) {
         shellExec(`cd ${buildBasePath} && underpost clone ${process.env.GITHUB_USERNAME}/${repoName}`);
         shellExec(`mkdir -p ${buildBasePath}/engine`);

package/src/server.js

@@ -4,7 +4,7 @@
 // https://expressjs.com/en/4x/api.html
 
 import { loggerFactory } from './server/logger.js';
-import { buildClient } from './server/client-build.js';
+import { buildClient } from './client-builder/client-build.js';
 import { buildRuntime } from './server/runtime.js';
 import { ProcessController } from './server/process.js';
 import { Config } from './server/conf.js';

package/test/deploy-monitor.test.js

@@ -108,6 +108,10 @@ if [[ "$verb" == "port-forward" ]]; then
   sleep 30
   exit 0
 fi
+if [[ "$verb" == "exec" ]]; then
+  grep -E '^container-status=' "$UNDERPOST_ENV_FILE" 2>/dev/null | tail -n1 | sed -E 's/^container-status=//'
+  exit 0
+fi
 exit 0
 `,
     );
@@ -118,10 +122,13 @@ exit 0
     fs.writeFileSync(
       monitorScriptPath,
       `import Underpost from ${JSON.stringify(pathToFileURL(path.join(repoRoot, 'src/index.js')).href)};
+const options = {};
+if (process.env.MON_READY_GATE) options.readyGate = process.env.MON_READY_GATE;
+if (process.env.MON_TRANSPORT) options.statusTransport = process.env.MON_TRANSPORT;
 try {
   await Underpost.monitor.monitorReadyRunner(${JSON.stringify(DEPLOY_ID)}, ${JSON.stringify(ENV)}, ${JSON.stringify(
     TRAFFIC,
-  )}, [], 'default');
+  )}, [], 'default', options);
   process.exit(0);
 } catch (_) {
   process.exit(1);
@@ -174,12 +181,18 @@ try {
       });
     });
 
-  it('success: both phases satisfied → monitor exits 0', async () => {
+  it('success (default exec transport): both phases satisfied → monitor exits 0', async () => {
     Underpost.env.set('container-status', RUNNING_STATUS);
     const code = await spawnMonitor({ FAKE_POD_READY: 'True' });
     expect(code).to.equal(0);
   });
 
+  it('success (opt-in http transport): both phases satisfied → monitor exits 0', async () => {
+    Underpost.env.set('container-status', RUNNING_STATUS);
+    const code = await spawnMonitor({ FAKE_POD_READY: 'True', MON_TRANSPORT: 'http' });
+    expect(code).to.equal(0);
+  });
+
   it('runtime failure: container-status=error → monitor exits 1', async () => {
     const monitorExit = spawnMonitor({ FAKE_POD_READY: 'True' });
     Underpost.env.set('container-status', 'error');
@@ -193,11 +206,12 @@ try {
     expect(code).to.equal(1);
   });
 
-  it('transport failure: endpoint unreachable is never success (exits 1)', async () => {
-    // Point the monitor at a port with no internal server; the HTTP read always
-    // fails, so runtime readiness is never confirmed and the monitor times out.
+  it('transport failure (http): endpoint unreachable is never success (exits 1)', async () => {
+    // Opt into http and point the monitor at a port with no internal server; the
+    // HTTP read always fails, so runtime readiness is never confirmed → timeout.
     Underpost.env.set('container-status', RUNNING_STATUS);
     const code = await spawnMonitor({
+      MON_TRANSPORT: 'http',
       UNDERPOST_INTERNAL_PORT: String(CLOSED_PORT),
       UNDERPOST_PF_LOCAL_PORT: String(CLOSED_PORT),
       UNDERPOST_MONITOR_MAX_ITERATIONS: '3',
@@ -220,4 +234,18 @@ try {
     Underpost.env.set('container-status', BUILD_STATUS);
     expect(await monitorExit).to.equal(1);
   });
+
+  // Custom instances (cyberia-*) gate on K8s Ready and read status via exec;
+  // their runtime never stamps `running-deployment` (stays `initializing`).
+  it('instance (kubernetes gate + exec): K8s Ready with initializing status → exits 0', async () => {
+    Underpost.env.set('container-status', INIT_STATUS);
+    const code = await spawnMonitor({ FAKE_POD_READY: 'True', MON_READY_GATE: 'kubernetes', MON_TRANSPORT: 'exec' });
+    expect(code).to.equal(0);
+  });
+
+  it('instance (kubernetes gate + exec): container-status=error → exits 1', async () => {
+    const monitorExit = spawnMonitor({ FAKE_POD_READY: 'True', MON_READY_GATE: 'kubernetes', MON_TRANSPORT: 'exec' });
+    Underpost.env.set('container-status', 'error');
+    expect(await monitorExit).to.equal(1);
+  });
 });

package/typedoc.json

@@ -8,7 +8,9 @@
     "./src/ws",
     "./src/grpc",
     "./src/mailer",
-    "./src/runtime"
+    "./src/runtime",
+    "./src/client-builder",
+    "./src/projects/underpost"
   ],
   "entryPointStrategy": "expand",
   "exclude": ["**/node_modules/**", "**/docs/**", "**/client/**"],