underpost 3.2.12 → 3.2.21

package/src/server/runtime-status.js

@@ -0,0 +1,235 @@
+/**
+ * Runtime status contract and the in-pod internal status endpoint.
+ *
+ * Single source of truth for the Underpost runtime readiness signal (Phase 2 of
+ * the two-phase deployment monitor). The runtime publishes its lifecycle here;
+ * the CD-side monitor (`src/cli/monitor.js`) reads it over HTTP via
+ * `kubectl port-forward`. Kubernetes pod readiness (Phase 1) is owned by kubelet
+ * and is intentionally not modeled in this module.
+ *
+ * Cross-process contract:
+ *   - In-pod, the canonical value lives in the underpost root env key
+ *     `container-status`, written by `start.js`. For non-error phases it carries
+ *     the namespaced form `<deployId>-<env>-<phase>`; a fatal fault collapses to
+ *     the bare value `error`.
+ *   - The internal HTTP server exposes that value (normalized to the bare
+ *     contract phase) and never exposes secrets, env dumps, or configuration.
+ *
+ * @module src/server/runtime-status.js
+ * @namespace RuntimeStatus
+ */
+
+import http from 'node:http';
+import fs from 'fs-extra';
+import dotenv from 'dotenv';
+import Underpost from '../index.js';
+import { loggerFactory } from './logger.js';
+
+const logger = loggerFactory(import.meta);
+
+/**
+ * Allowed runtime status contract values. These are the only Phase-2 signals
+ * the monitor reasons about.
+ * @memberof RuntimeStatus
+ */
+const RUNTIME_STATUS = {
+  BUILD: 'build-deployment',
+  INIT: 'initializing-deployment',
+  RUNNING: 'running-deployment',
+  ERROR: 'error',
+};
+
+const CONTAINER_STATUS_KEY = 'container-status';
+const INTERNAL_STATUS_PATH = '/_internal/status';
+const INTERNAL_READY_PATH = '/_internal/ready';
+const INTERNAL_HEALTH_PATH = '/_internal/health';
+
+/**
+ * Resolves the internal status port. Defaults to the deployment base `PORT`
+ * (app instances bind `PORT + 1` upward, so the base port is free inside the
+ * pod). An explicit `UNDERPOST_INTERNAL_PORT` override wins.
+ * @memberof RuntimeStatus
+ * @returns {number|undefined}
+ */
+const resolveInternalStatusPort = () => {
+  const raw = process.env.UNDERPOST_INTERNAL_PORT || process.env.PORT;
+  const port = parseInt(raw);
+  return Number.isNaN(port) ? undefined : port;
+};
+
+/**
+ * Single source of truth for the internal status port of a specific deployment,
+ * used identically by the in-pod server bind (`start.js`) and the CD-side
+ * monitor target (`monitor.js`) so the two can never disagree.
+ *
+ * Resolution order: `UNDERPOST_INTERNAL_PORT` override → the deployment's
+ * `.env.<env>` `PORT` → the ambient `PORT`.
+ *
+ * @memberof RuntimeStatus
+ * @param {string} deployId
+ * @param {string} env
+ * @returns {number|undefined}
+ */
+const deployStatusPort = (deployId, env) => {
+  const override = parseInt(process.env.UNDERPOST_INTERNAL_PORT);
+  if (!Number.isNaN(override)) return override;
+  try {
+    const envPath = `./engine-private/conf/${deployId}/.env.${env}`;
+    if (fs.existsSync(envPath)) {
+      const port = parseInt(dotenv.parse(fs.readFileSync(envPath, 'utf8')).PORT);
+      if (!Number.isNaN(port)) return port;
+    }
+  } catch (_) {
+    /* fall through to ambient resolution */
+  }
+  return resolveInternalStatusPort();
+};
+
+/**
+ * Builds the `container-status` env value for a lifecycle phase.
+ * @memberof RuntimeStatus
+ */
+const containerStatusValue = (deployId, env, phase) =>
+  phase === RUNTIME_STATUS.ERROR ? RUNTIME_STATUS.ERROR : `${deployId}-${env}-${phase}`;
+
+/**
+ * Normalizes a raw `container-status` value to a bare contract phase.
+ * Strips the `<deployId>-<env>-` prefix; `error` and unknown/empty values are
+ * passed through (empty → undefined).
+ * @memberof RuntimeStatus
+ * @param {string} raw
+ * @returns {string|undefined}
+ */
+const normalizeContainerStatus = (raw) => {
+  if (!raw || typeof raw !== 'string') return undefined;
+  const value = raw.trim();
+  if (!value || value === 'undefined' || value.toLowerCase().includes('empty')) return undefined;
+  if (value === RUNTIME_STATUS.ERROR) return RUNTIME_STATUS.ERROR;
+  for (const phase of [RUNTIME_STATUS.BUILD, RUNTIME_STATUS.INIT, RUNTIME_STATUS.RUNNING])
+    if (value.endsWith(`-${phase}`)) return phase;
+  return value;
+};
+
+/**
+ * Reads the current normalized runtime status from the env file.
+ * @memberof RuntimeStatus
+ * @returns {string|undefined}
+ */
+const getRuntimeStatus = () =>
+  normalizeContainerStatus(Underpost.env.get(CONTAINER_STATUS_KEY, undefined, { disableLog: true }));
+
+/**
+ * Minimal, secret-free payload served by the internal status endpoint and used
+ * by the monitor for failure classification and observability.
+ * @memberof RuntimeStatus
+ * @returns {{status: (string|null), deployId: (string|null), env: (string|null)}}
+ */
+const runtimeStatusPayload = () => ({
+  status: getRuntimeStatus() ?? null,
+  deployId: process.env.DEPLOY_ID ?? null,
+  env: process.env.NODE_ENV ?? null,
+});
+
+/**
+ * Emits a structured, secret-free deployment transition event.
+ * @memberof RuntimeStatus
+ */
+const emitRuntimeEvent = ({ deployId, env, phase }) => {
+  logger.info('runtime-status', {
+    deployId,
+    env,
+    phase: 'runtime',
+    status: phase,
+    timestamp: new Date().toISOString(),
+  });
+};
+
+/**
+ * Publishes a runtime lifecycle phase to the cross-process contract.
+ * @memberof RuntimeStatus
+ * @param {string} deployId
+ * @param {string} env
+ * @param {string} phase - One of {@link RUNTIME_STATUS}.
+ */
+const setRuntimeStatus = (deployId, env, phase) => {
+  Underpost.env.set(CONTAINER_STATUS_KEY, containerStatusValue(deployId, env, phase));
+  emitRuntimeEvent({ deployId, env, phase });
+};
+
+let internalServer;
+
+/**
+ * Starts the in-pod internal status server. Idempotent: repeated calls return
+ * the already-listening server. Exposes only the three internal endpoints and
+ * never serves secrets or configuration.
+ *
+ *   GET /_internal/status  → 200, `{status, deployId, env}` (monitor transport)
+ *   GET /_internal/ready   → 200 iff running-deployment, else 503 (readinessProbe)
+ *   GET /_internal/health  → 200 while the process is alive (livenessProbe)
+ *
+ * @memberof RuntimeStatus
+ * @param {number} [port]
+ * @returns {import('node:http').Server|undefined}
+ */
+const startInternalStatusServer = (port = resolveInternalStatusPort()) => {
+  if (internalServer) return internalServer;
+  if (!port) {
+    logger.warn('Internal status server not started: no resolvable port');
+    return undefined;
+  }
+  const server = http.createServer((req, res) => {
+    const url = (req.url || '').split('?')[0];
+    const sendJson = (code, body) => {
+      res.writeHead(code, { 'Content-Type': 'application/json' });
+      res.end(JSON.stringify(body));
+    };
+    if (req.method !== 'GET') return sendJson(405, { error: 'method_not_allowed' });
+    switch (url) {
+      case INTERNAL_HEALTH_PATH:
+        return sendJson(200, { status: 'ok' });
+      case INTERNAL_READY_PATH:
+        return getRuntimeStatus() === RUNTIME_STATUS.RUNNING
+          ? sendJson(200, { status: RUNTIME_STATUS.RUNNING })
+          : sendJson(503, { status: getRuntimeStatus() ?? null });
+      case INTERNAL_STATUS_PATH:
+        return sendJson(200, runtimeStatusPayload());
+      default:
+        return sendJson(404, { error: 'not_found' });
+    }
+  });
+  server.on('error', (error) => logger.error('internal status server error', error?.message ?? error));
+  server.listen(port, () => logger.info(`Internal status endpoint listening on :${port}${INTERNAL_STATUS_PATH}`));
+  internalServer = server;
+  return internalServer;
+};
+
+/**
+ * Stops the internal status server if running. Returns a promise that resolves
+ * once the listener is closed. Primarily a test/teardown hook.
+ * @memberof RuntimeStatus
+ * @returns {Promise<void>}
+ */
+const stopInternalStatusServer = () =>
+  new Promise((resolve) => {
+    if (!internalServer) return resolve();
+    const server = internalServer;
+    internalServer = undefined;
+    server.close(() => resolve());
+  });
+
+export {
+  RUNTIME_STATUS,
+  CONTAINER_STATUS_KEY,
+  INTERNAL_STATUS_PATH,
+  INTERNAL_READY_PATH,
+  INTERNAL_HEALTH_PATH,
+  resolveInternalStatusPort,
+  deployStatusPort,
+  containerStatusValue,
+  normalizeContainerStatus,
+  getRuntimeStatus,
+  runtimeStatusPayload,
+  setRuntimeStatus,
+  startInternalStatusServer,
+  stopInternalStatusServer,
+};

package/src/server/start.js

@@ -8,6 +8,7 @@ import fs from 'fs-extra';
 import { awaitDeployMonitor } from './conf.js';
 import { actionInitLog, loggerFactory } from './logger.js';
 import { shellCd, shellExec } from './process.js';
+import { RUNTIME_STATUS, setRuntimeStatus, startInternalStatusServer, deployStatusPort } from './runtime-status.js';
 import Underpost from '../index.js';
 const logger = loggerFactory(import.meta);
 
@@ -147,9 +148,13 @@ class UnderpostStartUp {
         pullBundle: false,
       },
     ) {
-      Underpost.env.set('container-status', `${deployId}-${env}-build-deployment`);
+      // Bring the internal status endpoint up first so Phase-2 readiness is
+      // observable through every lifecycle phase, including build and init. Bind
+      // the deployment-resolved port so it always matches the monitor's target.
+      startInternalStatusServer(deployStatusPort(deployId, env));
+      setRuntimeStatus(deployId, env, RUNTIME_STATUS.BUILD);
       if (options.build === true) await Underpost.start.build(deployId, env, options);
-      Underpost.env.set('container-status', `${deployId}-${env}-initializing-deployment`);
+      setRuntimeStatus(deployId, env, RUNTIME_STATUS.INIT);
       if (options.run === true) await Underpost.start.run(deployId, env, options);
     },
     /**
@@ -201,7 +206,7 @@ class UnderpostStartUp {
       const makeDeployCallback = (cmd) => (code, out, msg) => {
         if (code !== 0) {
           logger.error(`Deployment process exited with code ${code}`, { cmd, msg });
-          Underpost.env.set('container-status', 'error');
+          setRuntimeStatus(deployId, env, RUNTIME_STATUS.ERROR);
         }
       };
       if (fs.existsSync(`./engine-private/replica`)) {
@@ -211,18 +216,22 @@ class UnderpostStartUp {
           shellExec(`node bin env ${replica} ${env}`);
           const replicaCmd = `npm ${runCmd} ${replica}`;
           shellExec(replicaCmd, { async: true, callback: makeDeployCallback(replicaCmd) });
-          await awaitDeployMonitor();
+          const result = await awaitDeployMonitor();
+          if (result !== true) {
+            setRuntimeStatus(deployId, env, RUNTIME_STATUS.ERROR);
+            return;
+          }
         }
       }
       shellExec(`node bin env ${deployId} ${env}`);
       const deployCmd = `npm ${runCmd} ${deployId}`;
       shellExec(deployCmd, { async: true, callback: makeDeployCallback(deployCmd) });
-      await awaitDeployMonitor(true);
-      if (Underpost.env.get('container-status') !== 'error') {
+      const result = await awaitDeployMonitor(true);
+      if (result === true) {
         if (env === 'production' && Underpost.env.isInsideContainer()) Underpost.secret.globalSecretClean();
-        Underpost.env.set('container-status', `${deployId}-${env}-running-deployment`);
+        setRuntimeStatus(deployId, env, RUNTIME_STATUS.RUNNING);
       } else {
-        Underpost.env.set('container-status', 'error');
+        setRuntimeStatus(deployId, env, RUNTIME_STATUS.ERROR);
       }
     },
   };

package/test/deploy-monitor.test.js

@@ -0,0 +1,223 @@
+/**
+ * @module deploy-monitor.test
+ * @description End-to-end test of the two-phase deployment readiness contract
+ * between the in-pod runtime (`start.js` / `runtime-status.js`) and the CD-runner
+ * monitor (`Underpost.monitor.monitorReadyRunner`), exercised as real OS
+ * processes coordinating through the shared underpost env file and a real HTTP
+ * internal status endpoint.
+ *
+ * Contract under test:
+ *
+ *   - The in-pod runtime publishes only `container-status` (Phase 2) and serves
+ *     it over `GET /_internal/status`. It never propagates an exit code.
+ *
+ *   - monitorReadyRunner (CD runner) confirms BOTH phases before exiting 0:
+ *       Phase 1 — Kubernetes pod `Ready` condition (kubectl get pod -o json).
+ *       Phase 2 — runtime `running-deployment` read over HTTP via
+ *                 `kubectl port-forward` to the internal endpoint.
+ *     It throws (→ exit 1) on explicit runtime `error`, and returns (→ exit 0)
+ *     only once both phases are satisfied.
+ *
+ * The cluster surface is supplied by fake `sudo`/`kubectl` binaries on PATH:
+ *   - `get pods` / `get pod -o json` report one pod whose Ready condition is
+ *     driven by FAKE_POD_READY.
+ *   - `port-forward` is a no-op sleep; the monitor's HTTP GET reaches a REAL
+ *     internal status server bound in this test process (localPort == port),
+ *     which reads the same env file the runtime writes.
+ *
+ * The env file is redirected under a temp `npm_config_prefix`, so the test
+ * needs no cluster, no root, and never touches the machine's global install.
+ *
+ * Uses 'chai' for assertions.
+ */
+
+import { expect } from 'chai';
+import fs from 'fs-extra';
+import os from 'node:os';
+import path from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+import Underpost from '../src/index.js';
+import { shellExec } from '../src/server/process.js';
+import { startInternalStatusServer, stopInternalStatusServer } from '../src/server/runtime-status.js';
+
+const node = process.execPath;
+const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+
+const DEPLOY_ID = 'dd-test';
+const ENV = 'production';
+const TRAFFIC = 'green';
+const POD_NAME = `${DEPLOY_ID}-${ENV}-${TRAFFIC}-pod`;
+const RUNNING_STATUS = `${DEPLOY_ID}-${ENV}-running-deployment`;
+const INIT_STATUS = `${DEPLOY_ID}-${ENV}-initializing-deployment`;
+const BUILD_STATUS = `${DEPLOY_ID}-${ENV}-build-deployment`;
+
+const INTERNAL_PORT = 39517; // internal status endpoint (real server bound here)
+const CLOSED_PORT = 39518; // no server — used to force a transport failure
+
+describe('Deploy monitor — two-phase state machine (e2e, real HTTP transport)', function () {
+  this.timeout(60000);
+
+  let prevPrefix;
+  let tmpPrefix;
+  let fakeBinDir;
+  let envFile;
+  let monitorScriptPath;
+
+  before(() => {
+    prevPrefix = process.env.npm_config_prefix;
+    tmpPrefix = fs.mkdtempSync(path.join(os.tmpdir(), 'underpost-e2e-'));
+    process.env.npm_config_prefix = tmpPrefix;
+
+    Underpost.env.set('container-status', 'init');
+    const npmRoot = shellExec('npm root -g', { stdout: true, silent: true, disableLog: true }).trim();
+    envFile = path.join(npmRoot, 'underpost', '.env');
+
+    // Real in-pod internal status server: serves container-status from the same
+    // env file the runtime writes. Bound in this test process; the monitor's
+    // port-forward tunnel (localPort == INTERNAL_PORT) resolves straight to it.
+    process.env.UNDERPOST_INTERNAL_PORT = String(INTERNAL_PORT);
+    startInternalStatusServer(INTERNAL_PORT);
+
+    fakeBinDir = path.join(tmpPrefix, 'fakebin');
+    fs.ensureDirSync(fakeBinDir);
+
+    const sudoPath = path.join(fakeBinDir, 'sudo');
+    fs.writeFileSync(
+      sudoPath,
+      `#!/usr/bin/env bash
+while [[ "$1" == -* || "$1" == "--" ]]; do shift; done
+exec "$@"
+`,
+    );
+
+    const kubectlPath = path.join(fakeBinDir, 'kubectl');
+    fs.writeFileSync(
+      kubectlPath,
+      `#!/usr/bin/env bash
+verb="$1"; kind="$2"
+if [[ "$verb" == "get" && "$kind" == "pods" ]]; then
+  printf 'NAME READY STATUS RESTARTS AGE\\n'
+  printf '%s 1/1 Running 0 1m\\n' "$POD_NAME"
+  exit 0
+fi
+if [[ "$verb" == "get" && "$kind" == "pod" ]]; then
+  printf '{"status":{"conditions":[{"type":"Ready","status":"%s"}]}}\\n' "\${FAKE_POD_READY:-True}"
+  exit 0
+fi
+if [[ "$verb" == "port-forward" ]]; then
+  sleep 30
+  exit 0
+fi
+exit 0
+`,
+    );
+    fs.chmodSync(sudoPath, 0o755);
+    fs.chmodSync(kubectlPath, 0o755);
+
+    monitorScriptPath = path.join(tmpPrefix, 'monitor-ready.mjs');
+    fs.writeFileSync(
+      monitorScriptPath,
+      `import Underpost from ${JSON.stringify(pathToFileURL(path.join(repoRoot, 'src/index.js')).href)};
+try {
+  await Underpost.monitor.monitorReadyRunner(${JSON.stringify(DEPLOY_ID)}, ${JSON.stringify(ENV)}, ${JSON.stringify(
+    TRAFFIC,
+  )}, [], 'default');
+  process.exit(0);
+} catch (_) {
+  process.exit(1);
+}
+`,
+    );
+  });
+
+  after(async () => {
+    await stopInternalStatusServer();
+    delete process.env.UNDERPOST_INTERNAL_PORT;
+    if (prevPrefix === undefined) delete process.env.npm_config_prefix;
+    else process.env.npm_config_prefix = prevPrefix;
+    fs.removeSync(tmpPrefix);
+  });
+
+  beforeEach(() => {
+    // Deploy in flight: app not yet reporting running.
+    Underpost.env.set('container-status', INIT_STATUS);
+  });
+
+  // Spawns the real monitorReadyRunner with the fake cluster on PATH; resolves
+  // with its exit code. `overrides` inject deterministic timing / target port.
+  const spawnMonitor = (overrides = {}) =>
+    new Promise((resolve) => {
+      const envVars = {
+        PATH: `${fakeBinDir}:${process.env.PATH}`,
+        UNDERPOST_ENV_FILE: envFile,
+        POD_NAME,
+        npm_config_prefix: tmpPrefix,
+        UNDERPOST_INTERNAL_PORT: String(INTERNAL_PORT),
+        // Pin the tunnel's local port so the no-op fake port-forward + the real
+        // internal server (bound to INTERNAL_PORT in this process) resolve to the
+        // same address the monitor's HTTP GET targets.
+        UNDERPOST_PF_LOCAL_PORT: String(INTERNAL_PORT),
+        UNDERPOST_MONITOR_DELAY_MS: '100',
+        UNDERPOST_MONITOR_MAX_ITERATIONS: '60',
+        UNDERPOST_PF_ATTEMPTS: '3',
+        FAKE_POD_READY: 'True',
+        ...overrides,
+      };
+      const prefix = Object.entries(envVars)
+        .map(([k, v]) => `${k}="${v}"`)
+        .join(' ');
+      shellExec(`${prefix} ${node} ${monitorScriptPath}`, {
+        async: true,
+        silent: true,
+        disableLog: true,
+        callback: (code) => resolve(code),
+      });
+    });
+
+  it('success: both phases satisfied → monitor exits 0', async () => {
+    Underpost.env.set('container-status', RUNNING_STATUS);
+    const code = await spawnMonitor({ FAKE_POD_READY: 'True' });
+    expect(code).to.equal(0);
+  });
+
+  it('runtime failure: container-status=error → monitor exits 1', async () => {
+    const monitorExit = spawnMonitor({ FAKE_POD_READY: 'True' });
+    Underpost.env.set('container-status', 'error');
+    expect(await monitorExit).to.equal(1);
+  });
+
+  it('readiness mismatch: runtime running but pod not Ready → never succeeds (exits 1)', async () => {
+    // Phase 2 satisfied, Phase 1 not: success requires BOTH, so it must time out.
+    Underpost.env.set('container-status', RUNNING_STATUS);
+    const code = await spawnMonitor({ FAKE_POD_READY: 'False', UNDERPOST_MONITOR_MAX_ITERATIONS: '4' });
+    expect(code).to.equal(1);
+  });
+
+  it('transport failure: endpoint unreachable is never success (exits 1)', async () => {
+    // Point the monitor at a port with no internal server; the HTTP read always
+    // fails, so runtime readiness is never confirmed and the monitor times out.
+    Underpost.env.set('container-status', RUNNING_STATUS);
+    const code = await spawnMonitor({
+      UNDERPOST_INTERNAL_PORT: String(CLOSED_PORT),
+      UNDERPOST_PF_LOCAL_PORT: String(CLOSED_PORT),
+      UNDERPOST_MONITOR_MAX_ITERATIONS: '3',
+    });
+    expect(code).to.equal(1);
+  });
+
+  it('timeout: runtime stuck initializing → monitor exits 1', async () => {
+    Underpost.env.set('container-status', INIT_STATUS);
+    const code = await spawnMonitor({ FAKE_POD_READY: 'True', UNDERPOST_MONITOR_MAX_ITERATIONS: '4' });
+    expect(code).to.equal(1);
+  });
+
+  it('regression: advanced pod whose runtime status falls back → monitor exits 1', async () => {
+    // Pod advances past build, then its reported status regresses (pod restart);
+    // the monitor must treat this as a failure rather than wait it out.
+    Underpost.env.set('container-status', INIT_STATUS);
+    const monitorExit = spawnMonitor({ FAKE_POD_READY: 'False', UNDERPOST_MONITOR_MAX_ITERATIONS: '120' });
+    await new Promise((r) => setTimeout(r, 1500));
+    Underpost.env.set('container-status', BUILD_STATUS);
+    expect(await monitorExit).to.equal(1);
+  });
+});