underpost 3.2.14 → 3.2.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +57 -1
- package/CLI-HELP.md +97 -30
- package/README.md +2 -2
- package/bin/build.template.js +4 -2
- package/manifests/cronjobs/dd-cron/dd-cron-backup.yaml +1 -1
- package/manifests/cronjobs/dd-cron/dd-cron-dns.yaml +1 -1
- package/manifests/deployment/dd-default-development/deployment.yaml +2 -2
- package/package.json +1 -1
- package/scripts/link-local-underpost-cli.sh +6 -0
- package/scripts/test-monitor.sh +86 -0
- package/src/cli/deploy.js +195 -46
- package/src/cli/env.js +1 -4
- package/src/cli/index.js +39 -0
- package/src/cli/monitor.js +184 -72
- package/src/cli/release.js +21 -6
- package/src/cli/repository.js +18 -4
- package/src/cli/run.js +22 -0
- package/src/db/mongo/MongooseDB.js +2 -1
- package/src/index.js +1 -1
- package/src/server/conf.js +33 -9
- package/src/server/runtime-status.js +235 -0
- package/src/server/start.js +11 -6
- package/test/deploy-monitor.test.js +108 -73
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Runtime status contract and the in-pod internal status endpoint.
|
|
3
|
+
*
|
|
4
|
+
* Single source of truth for the Underpost runtime readiness signal (Phase 2 of
|
|
5
|
+
* the two-phase deployment monitor). The runtime publishes its lifecycle here;
|
|
6
|
+
* the CD-side monitor (`src/cli/monitor.js`) reads it over HTTP via
|
|
7
|
+
* `kubectl port-forward`. Kubernetes pod readiness (Phase 1) is owned by kubelet
|
|
8
|
+
* and is intentionally not modeled in this module.
|
|
9
|
+
*
|
|
10
|
+
* Cross-process contract:
|
|
11
|
+
* - In-pod, the canonical value lives in the underpost root env key
|
|
12
|
+
* `container-status`, written by `start.js`. For non-error phases it carries
|
|
13
|
+
* the namespaced form `<deployId>-<env>-<phase>`; a fatal fault collapses to
|
|
14
|
+
* the bare value `error`.
|
|
15
|
+
* - The internal HTTP server exposes that value (normalized to the bare
|
|
16
|
+
* contract phase) and never exposes secrets, env dumps, or configuration.
|
|
17
|
+
*
|
|
18
|
+
* @module src/server/runtime-status.js
|
|
19
|
+
* @namespace RuntimeStatus
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import http from 'node:http';
|
|
23
|
+
import fs from 'fs-extra';
|
|
24
|
+
import dotenv from 'dotenv';
|
|
25
|
+
import Underpost from '../index.js';
|
|
26
|
+
import { loggerFactory } from './logger.js';
|
|
27
|
+
|
|
28
|
+
const logger = loggerFactory(import.meta);
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Allowed runtime status contract values. These are the only Phase-2 signals
|
|
32
|
+
* the monitor reasons about.
|
|
33
|
+
* @memberof RuntimeStatus
|
|
34
|
+
*/
|
|
35
|
+
const RUNTIME_STATUS = {
|
|
36
|
+
BUILD: 'build-deployment',
|
|
37
|
+
INIT: 'initializing-deployment',
|
|
38
|
+
RUNNING: 'running-deployment',
|
|
39
|
+
ERROR: 'error',
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
const CONTAINER_STATUS_KEY = 'container-status';
|
|
43
|
+
const INTERNAL_STATUS_PATH = '/_internal/status';
|
|
44
|
+
const INTERNAL_READY_PATH = '/_internal/ready';
|
|
45
|
+
const INTERNAL_HEALTH_PATH = '/_internal/health';
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Resolves the internal status port. Defaults to the deployment base `PORT`
|
|
49
|
+
* (app instances bind `PORT + 1` upward, so the base port is free inside the
|
|
50
|
+
* pod). An explicit `UNDERPOST_INTERNAL_PORT` override wins.
|
|
51
|
+
* @memberof RuntimeStatus
|
|
52
|
+
* @returns {number|undefined}
|
|
53
|
+
*/
|
|
54
|
+
const resolveInternalStatusPort = () => {
|
|
55
|
+
const raw = process.env.UNDERPOST_INTERNAL_PORT || process.env.PORT;
|
|
56
|
+
const port = parseInt(raw);
|
|
57
|
+
return Number.isNaN(port) ? undefined : port;
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Single source of truth for the internal status port of a specific deployment,
|
|
62
|
+
* used identically by the in-pod server bind (`start.js`) and the CD-side
|
|
63
|
+
* monitor target (`monitor.js`) so the two can never disagree.
|
|
64
|
+
*
|
|
65
|
+
* Resolution order: `UNDERPOST_INTERNAL_PORT` override → the deployment's
|
|
66
|
+
* `.env.<env>` `PORT` → the ambient `PORT`.
|
|
67
|
+
*
|
|
68
|
+
* @memberof RuntimeStatus
|
|
69
|
+
* @param {string} deployId
|
|
70
|
+
* @param {string} env
|
|
71
|
+
* @returns {number|undefined}
|
|
72
|
+
*/
|
|
73
|
+
const deployStatusPort = (deployId, env) => {
|
|
74
|
+
const override = parseInt(process.env.UNDERPOST_INTERNAL_PORT);
|
|
75
|
+
if (!Number.isNaN(override)) return override;
|
|
76
|
+
try {
|
|
77
|
+
const envPath = `./engine-private/conf/${deployId}/.env.${env}`;
|
|
78
|
+
if (fs.existsSync(envPath)) {
|
|
79
|
+
const port = parseInt(dotenv.parse(fs.readFileSync(envPath, 'utf8')).PORT);
|
|
80
|
+
if (!Number.isNaN(port)) return port;
|
|
81
|
+
}
|
|
82
|
+
} catch (_) {
|
|
83
|
+
/* fall through to ambient resolution */
|
|
84
|
+
}
|
|
85
|
+
return resolveInternalStatusPort();
|
|
86
|
+
};
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Builds the `container-status` env value for a lifecycle phase.
|
|
90
|
+
* @memberof RuntimeStatus
|
|
91
|
+
*/
|
|
92
|
+
const containerStatusValue = (deployId, env, phase) =>
|
|
93
|
+
phase === RUNTIME_STATUS.ERROR ? RUNTIME_STATUS.ERROR : `${deployId}-${env}-${phase}`;
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Normalizes a raw `container-status` value to a bare contract phase.
|
|
97
|
+
* Strips the `<deployId>-<env>-` prefix; `error` and unknown/empty values are
|
|
98
|
+
* passed through (empty → undefined).
|
|
99
|
+
* @memberof RuntimeStatus
|
|
100
|
+
* @param {string} raw
|
|
101
|
+
* @returns {string|undefined}
|
|
102
|
+
*/
|
|
103
|
+
const normalizeContainerStatus = (raw) => {
|
|
104
|
+
if (!raw || typeof raw !== 'string') return undefined;
|
|
105
|
+
const value = raw.trim();
|
|
106
|
+
if (!value || value === 'undefined' || value.toLowerCase().includes('empty')) return undefined;
|
|
107
|
+
if (value === RUNTIME_STATUS.ERROR) return RUNTIME_STATUS.ERROR;
|
|
108
|
+
for (const phase of [RUNTIME_STATUS.BUILD, RUNTIME_STATUS.INIT, RUNTIME_STATUS.RUNNING])
|
|
109
|
+
if (value.endsWith(`-${phase}`)) return phase;
|
|
110
|
+
return value;
|
|
111
|
+
};
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Reads the current normalized runtime status from the env file.
|
|
115
|
+
* @memberof RuntimeStatus
|
|
116
|
+
* @returns {string|undefined}
|
|
117
|
+
*/
|
|
118
|
+
const getRuntimeStatus = () =>
|
|
119
|
+
normalizeContainerStatus(Underpost.env.get(CONTAINER_STATUS_KEY, undefined, { disableLog: true }));
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Minimal, secret-free payload served by the internal status endpoint and used
|
|
123
|
+
* by the monitor for failure classification and observability.
|
|
124
|
+
* @memberof RuntimeStatus
|
|
125
|
+
* @returns {{status: (string|null), deployId: (string|null), env: (string|null)}}
|
|
126
|
+
*/
|
|
127
|
+
const runtimeStatusPayload = () => ({
|
|
128
|
+
status: getRuntimeStatus() ?? null,
|
|
129
|
+
deployId: process.env.DEPLOY_ID ?? null,
|
|
130
|
+
env: process.env.NODE_ENV ?? null,
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Emits a structured, secret-free deployment transition event.
|
|
135
|
+
* @memberof RuntimeStatus
|
|
136
|
+
*/
|
|
137
|
+
const emitRuntimeEvent = ({ deployId, env, phase }) => {
|
|
138
|
+
logger.info('runtime-status', {
|
|
139
|
+
deployId,
|
|
140
|
+
env,
|
|
141
|
+
phase: 'runtime',
|
|
142
|
+
status: phase,
|
|
143
|
+
timestamp: new Date().toISOString(),
|
|
144
|
+
});
|
|
145
|
+
};
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* Publishes a runtime lifecycle phase to the cross-process contract.
|
|
149
|
+
* @memberof RuntimeStatus
|
|
150
|
+
* @param {string} deployId
|
|
151
|
+
* @param {string} env
|
|
152
|
+
* @param {string} phase - One of {@link RUNTIME_STATUS}.
|
|
153
|
+
*/
|
|
154
|
+
const setRuntimeStatus = (deployId, env, phase) => {
|
|
155
|
+
Underpost.env.set(CONTAINER_STATUS_KEY, containerStatusValue(deployId, env, phase));
|
|
156
|
+
emitRuntimeEvent({ deployId, env, phase });
|
|
157
|
+
};
|
|
158
|
+
|
|
159
|
+
let internalServer;
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* Starts the in-pod internal status server. Idempotent: repeated calls return
|
|
163
|
+
* the already-listening server. Exposes only the three internal endpoints and
|
|
164
|
+
* never serves secrets or configuration.
|
|
165
|
+
*
|
|
166
|
+
* GET /_internal/status → 200, `{status, deployId, env}` (monitor transport)
|
|
167
|
+
* GET /_internal/ready → 200 iff running-deployment, else 503 (readinessProbe)
|
|
168
|
+
* GET /_internal/health → 200 while the process is alive (livenessProbe)
|
|
169
|
+
*
|
|
170
|
+
* @memberof RuntimeStatus
|
|
171
|
+
* @param {number} [port]
|
|
172
|
+
* @returns {import('node:http').Server|undefined}
|
|
173
|
+
*/
|
|
174
|
+
const startInternalStatusServer = (port = resolveInternalStatusPort()) => {
|
|
175
|
+
if (internalServer) return internalServer;
|
|
176
|
+
if (!port) {
|
|
177
|
+
logger.warn('Internal status server not started: no resolvable port');
|
|
178
|
+
return undefined;
|
|
179
|
+
}
|
|
180
|
+
const server = http.createServer((req, res) => {
|
|
181
|
+
const url = (req.url || '').split('?')[0];
|
|
182
|
+
const sendJson = (code, body) => {
|
|
183
|
+
res.writeHead(code, { 'Content-Type': 'application/json' });
|
|
184
|
+
res.end(JSON.stringify(body));
|
|
185
|
+
};
|
|
186
|
+
if (req.method !== 'GET') return sendJson(405, { error: 'method_not_allowed' });
|
|
187
|
+
switch (url) {
|
|
188
|
+
case INTERNAL_HEALTH_PATH:
|
|
189
|
+
return sendJson(200, { status: 'ok' });
|
|
190
|
+
case INTERNAL_READY_PATH:
|
|
191
|
+
return getRuntimeStatus() === RUNTIME_STATUS.RUNNING
|
|
192
|
+
? sendJson(200, { status: RUNTIME_STATUS.RUNNING })
|
|
193
|
+
: sendJson(503, { status: getRuntimeStatus() ?? null });
|
|
194
|
+
case INTERNAL_STATUS_PATH:
|
|
195
|
+
return sendJson(200, runtimeStatusPayload());
|
|
196
|
+
default:
|
|
197
|
+
return sendJson(404, { error: 'not_found' });
|
|
198
|
+
}
|
|
199
|
+
});
|
|
200
|
+
server.on('error', (error) => logger.error('internal status server error', error?.message ?? error));
|
|
201
|
+
server.listen(port, () => logger.info(`Internal status endpoint listening on :${port}${INTERNAL_STATUS_PATH}`));
|
|
202
|
+
internalServer = server;
|
|
203
|
+
return internalServer;
|
|
204
|
+
};
|
|
205
|
+
|
|
206
|
+
/**
|
|
207
|
+
* Stops the internal status server if running. Returns a promise that resolves
|
|
208
|
+
* once the listener is closed. Primarily a test/teardown hook.
|
|
209
|
+
* @memberof RuntimeStatus
|
|
210
|
+
* @returns {Promise<void>}
|
|
211
|
+
*/
|
|
212
|
+
const stopInternalStatusServer = () =>
|
|
213
|
+
new Promise((resolve) => {
|
|
214
|
+
if (!internalServer) return resolve();
|
|
215
|
+
const server = internalServer;
|
|
216
|
+
internalServer = undefined;
|
|
217
|
+
server.close(() => resolve());
|
|
218
|
+
});
|
|
219
|
+
|
|
220
|
+
export {
|
|
221
|
+
RUNTIME_STATUS,
|
|
222
|
+
CONTAINER_STATUS_KEY,
|
|
223
|
+
INTERNAL_STATUS_PATH,
|
|
224
|
+
INTERNAL_READY_PATH,
|
|
225
|
+
INTERNAL_HEALTH_PATH,
|
|
226
|
+
resolveInternalStatusPort,
|
|
227
|
+
deployStatusPort,
|
|
228
|
+
containerStatusValue,
|
|
229
|
+
normalizeContainerStatus,
|
|
230
|
+
getRuntimeStatus,
|
|
231
|
+
runtimeStatusPayload,
|
|
232
|
+
setRuntimeStatus,
|
|
233
|
+
startInternalStatusServer,
|
|
234
|
+
stopInternalStatusServer,
|
|
235
|
+
};
|
package/src/server/start.js
CHANGED
|
@@ -8,6 +8,7 @@ import fs from 'fs-extra';
|
|
|
8
8
|
import { awaitDeployMonitor } from './conf.js';
|
|
9
9
|
import { actionInitLog, loggerFactory } from './logger.js';
|
|
10
10
|
import { shellCd, shellExec } from './process.js';
|
|
11
|
+
import { RUNTIME_STATUS, setRuntimeStatus, startInternalStatusServer, deployStatusPort } from './runtime-status.js';
|
|
11
12
|
import Underpost from '../index.js';
|
|
12
13
|
const logger = loggerFactory(import.meta);
|
|
13
14
|
|
|
@@ -147,9 +148,13 @@ class UnderpostStartUp {
|
|
|
147
148
|
pullBundle: false,
|
|
148
149
|
},
|
|
149
150
|
) {
|
|
150
|
-
|
|
151
|
+
// Bring the internal status endpoint up first so Phase-2 readiness is
|
|
152
|
+
// observable through every lifecycle phase, including build and init. Bind
|
|
153
|
+
// the deployment-resolved port so it always matches the monitor's target.
|
|
154
|
+
startInternalStatusServer(deployStatusPort(deployId, env));
|
|
155
|
+
setRuntimeStatus(deployId, env, RUNTIME_STATUS.BUILD);
|
|
151
156
|
if (options.build === true) await Underpost.start.build(deployId, env, options);
|
|
152
|
-
|
|
157
|
+
setRuntimeStatus(deployId, env, RUNTIME_STATUS.INIT);
|
|
153
158
|
if (options.run === true) await Underpost.start.run(deployId, env, options);
|
|
154
159
|
},
|
|
155
160
|
/**
|
|
@@ -201,7 +206,7 @@ class UnderpostStartUp {
|
|
|
201
206
|
const makeDeployCallback = (cmd) => (code, out, msg) => {
|
|
202
207
|
if (code !== 0) {
|
|
203
208
|
logger.error(`Deployment process exited with code ${code}`, { cmd, msg });
|
|
204
|
-
|
|
209
|
+
setRuntimeStatus(deployId, env, RUNTIME_STATUS.ERROR);
|
|
205
210
|
}
|
|
206
211
|
};
|
|
207
212
|
if (fs.existsSync(`./engine-private/replica`)) {
|
|
@@ -213,7 +218,7 @@ class UnderpostStartUp {
|
|
|
213
218
|
shellExec(replicaCmd, { async: true, callback: makeDeployCallback(replicaCmd) });
|
|
214
219
|
const result = await awaitDeployMonitor();
|
|
215
220
|
if (result !== true) {
|
|
216
|
-
|
|
221
|
+
setRuntimeStatus(deployId, env, RUNTIME_STATUS.ERROR);
|
|
217
222
|
return;
|
|
218
223
|
}
|
|
219
224
|
}
|
|
@@ -224,9 +229,9 @@ class UnderpostStartUp {
|
|
|
224
229
|
const result = await awaitDeployMonitor(true);
|
|
225
230
|
if (result === true) {
|
|
226
231
|
if (env === 'production' && Underpost.env.isInsideContainer()) Underpost.secret.globalSecretClean();
|
|
227
|
-
|
|
232
|
+
setRuntimeStatus(deployId, env, RUNTIME_STATUS.RUNNING);
|
|
228
233
|
} else {
|
|
229
|
-
|
|
234
|
+
setRuntimeStatus(deployId, env, RUNTIME_STATUS.ERROR);
|
|
230
235
|
}
|
|
231
236
|
},
|
|
232
237
|
};
|
|
@@ -1,28 +1,32 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @module deploy-monitor.test
|
|
3
|
-
* @description End-to-end test of the
|
|
4
|
-
* the in-pod runtime (start.js) and the CD-runner
|
|
5
|
-
* (`Underpost.monitor.monitorReadyRunner`), exercised as
|
|
6
|
-
*
|
|
3
|
+
* @description End-to-end test of the two-phase deployment readiness contract
|
|
4
|
+
* between the in-pod runtime (`start.js` / `runtime-status.js`) and the CD-runner
|
|
5
|
+
* monitor (`Underpost.monitor.monitorReadyRunner`), exercised as real OS
|
|
6
|
+
* processes coordinating through the shared underpost env file and a real HTTP
|
|
7
|
+
* internal status endpoint.
|
|
7
8
|
*
|
|
8
9
|
* Contract under test:
|
|
9
10
|
*
|
|
10
|
-
* -
|
|
11
|
-
*
|
|
12
|
-
* healthy one it sets `container-status=<deploy>-<env>-running-deployment`.
|
|
11
|
+
* - The in-pod runtime publishes only `container-status` (Phase 2) and serves
|
|
12
|
+
* it over `GET /_internal/status`. It never propagates an exit code.
|
|
13
13
|
*
|
|
14
|
-
* - monitorReadyRunner (CD runner)
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
* `
|
|
14
|
+
* - monitorReadyRunner (CD runner) confirms BOTH phases before exiting 0:
|
|
15
|
+
* Phase 1 — Kubernetes pod `Ready` condition (kubectl get pod -o json).
|
|
16
|
+
* Phase 2 — runtime `running-deployment` read over HTTP via
|
|
17
|
+
* `kubectl port-forward` to the internal endpoint.
|
|
18
|
+
* It throws (→ exit 1) on explicit runtime `error`, and returns (→ exit 0)
|
|
19
|
+
* only once both phases are satisfied.
|
|
19
20
|
*
|
|
20
|
-
* The cluster surface
|
|
21
|
-
* `
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
* the
|
|
21
|
+
* The cluster surface is supplied by fake `sudo`/`kubectl` binaries on PATH:
|
|
22
|
+
* - `get pods` / `get pod -o json` report one pod whose Ready condition is
|
|
23
|
+
* driven by FAKE_POD_READY.
|
|
24
|
+
* - `port-forward` is a no-op sleep; the monitor's HTTP GET reaches a REAL
|
|
25
|
+
* internal status server bound in this test process (localPort == port),
|
|
26
|
+
* which reads the same env file the runtime writes.
|
|
27
|
+
*
|
|
28
|
+
* The env file is redirected under a temp `npm_config_prefix`, so the test
|
|
29
|
+
* needs no cluster, no root, and never touches the machine's global install.
|
|
26
30
|
*
|
|
27
31
|
* Uses 'chai' for assertions.
|
|
28
32
|
*/
|
|
@@ -34,6 +38,7 @@ import path from 'node:path';
|
|
|
34
38
|
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
35
39
|
import Underpost from '../src/index.js';
|
|
36
40
|
import { shellExec } from '../src/server/process.js';
|
|
41
|
+
import { startInternalStatusServer, stopInternalStatusServer } from '../src/server/runtime-status.js';
|
|
37
42
|
|
|
38
43
|
const node = process.execPath;
|
|
39
44
|
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
|
@@ -43,9 +48,14 @@ const ENV = 'production';
|
|
|
43
48
|
const TRAFFIC = 'green';
|
|
44
49
|
const POD_NAME = `${DEPLOY_ID}-${ENV}-${TRAFFIC}-pod`;
|
|
45
50
|
const RUNNING_STATUS = `${DEPLOY_ID}-${ENV}-running-deployment`;
|
|
51
|
+
const INIT_STATUS = `${DEPLOY_ID}-${ENV}-initializing-deployment`;
|
|
52
|
+
const BUILD_STATUS = `${DEPLOY_ID}-${ENV}-build-deployment`;
|
|
53
|
+
|
|
54
|
+
const INTERNAL_PORT = 39517; // internal status endpoint (real server bound here)
|
|
55
|
+
const CLOSED_PORT = 39518; // no server — used to force a transport failure
|
|
46
56
|
|
|
47
|
-
describe('Deploy monitor —
|
|
48
|
-
this.timeout(
|
|
57
|
+
describe('Deploy monitor — two-phase state machine (e2e, real HTTP transport)', function () {
|
|
58
|
+
this.timeout(60000);
|
|
49
59
|
|
|
50
60
|
let prevPrefix;
|
|
51
61
|
let tmpPrefix;
|
|
@@ -58,15 +68,16 @@ describe('Deploy monitor — start.js ↔ monitorReadyRunner (e2e, parallel proc
|
|
|
58
68
|
tmpPrefix = fs.mkdtempSync(path.join(os.tmpdir(), 'underpost-e2e-'));
|
|
59
69
|
process.env.npm_config_prefix = tmpPrefix;
|
|
60
70
|
|
|
61
|
-
// Materialize the underpost env file and resolve its absolute path (the
|
|
62
|
-
// fake `kubectl exec` reads container-status straight from it).
|
|
63
71
|
Underpost.env.set('container-status', 'init');
|
|
64
72
|
const npmRoot = shellExec('npm root -g', { stdout: true, silent: true, disableLog: true }).trim();
|
|
65
73
|
envFile = path.join(npmRoot, 'underpost', '.env');
|
|
66
74
|
|
|
67
|
-
//
|
|
68
|
-
//
|
|
69
|
-
//
|
|
75
|
+
// Real in-pod internal status server: serves container-status from the same
|
|
76
|
+
// env file the runtime writes. Bound in this test process; the monitor's
|
|
77
|
+
// port-forward tunnel (localPort == INTERNAL_PORT) resolves straight to it.
|
|
78
|
+
process.env.UNDERPOST_INTERNAL_PORT = String(INTERNAL_PORT);
|
|
79
|
+
startInternalStatusServer(INTERNAL_PORT);
|
|
80
|
+
|
|
70
81
|
fakeBinDir = path.join(tmpPrefix, 'fakebin');
|
|
71
82
|
fs.ensureDirSync(fakeBinDir);
|
|
72
83
|
|
|
@@ -90,11 +101,11 @@ if [[ "$verb" == "get" && "$kind" == "pods" ]]; then
|
|
|
90
101
|
exit 0
|
|
91
102
|
fi
|
|
92
103
|
if [[ "$verb" == "get" && "$kind" == "pod" ]]; then
|
|
93
|
-
printf '{"status":{"conditions":[{"type":"Ready","status":"
|
|
104
|
+
printf '{"status":{"conditions":[{"type":"Ready","status":"%s"}]}}\\n' "\${FAKE_POD_READY:-True}"
|
|
94
105
|
exit 0
|
|
95
106
|
fi
|
|
96
|
-
if [[ "$verb" == "
|
|
97
|
-
|
|
107
|
+
if [[ "$verb" == "port-forward" ]]; then
|
|
108
|
+
sleep 30
|
|
98
109
|
exit 0
|
|
99
110
|
fi
|
|
100
111
|
exit 0
|
|
@@ -103,17 +114,14 @@ exit 0
|
|
|
103
114
|
fs.chmodSync(sudoPath, 0o755);
|
|
104
115
|
fs.chmodSync(kubectlPath, 0o755);
|
|
105
116
|
|
|
106
|
-
// Real monitorReadyRunner in its own process: exit 0 when it returns (ready),
|
|
107
|
-
// exit 1 when it throws (container-status=error). This is the signal `set -e`
|
|
108
|
-
// turns into a passed/failed GitHub Actions job.
|
|
109
117
|
monitorScriptPath = path.join(tmpPrefix, 'monitor-ready.mjs');
|
|
110
118
|
fs.writeFileSync(
|
|
111
119
|
monitorScriptPath,
|
|
112
120
|
`import Underpost from ${JSON.stringify(pathToFileURL(path.join(repoRoot, 'src/index.js')).href)};
|
|
113
121
|
try {
|
|
114
122
|
await Underpost.monitor.monitorReadyRunner(${JSON.stringify(DEPLOY_ID)}, ${JSON.stringify(ENV)}, ${JSON.stringify(
|
|
115
|
-
|
|
116
|
-
|
|
123
|
+
TRAFFIC,
|
|
124
|
+
)}, [], 'default');
|
|
117
125
|
process.exit(0);
|
|
118
126
|
} catch (_) {
|
|
119
127
|
process.exit(1);
|
|
@@ -122,26 +130,42 @@ try {
|
|
|
122
130
|
);
|
|
123
131
|
});
|
|
124
132
|
|
|
125
|
-
after(() => {
|
|
133
|
+
after(async () => {
|
|
134
|
+
await stopInternalStatusServer();
|
|
135
|
+
delete process.env.UNDERPOST_INTERNAL_PORT;
|
|
126
136
|
if (prevPrefix === undefined) delete process.env.npm_config_prefix;
|
|
127
137
|
else process.env.npm_config_prefix = prevPrefix;
|
|
128
138
|
fs.removeSync(tmpPrefix);
|
|
129
139
|
});
|
|
130
140
|
|
|
131
141
|
beforeEach(() => {
|
|
132
|
-
// Deploy in flight:
|
|
133
|
-
Underpost.env.set('container-status',
|
|
142
|
+
// Deploy in flight: app not yet reporting running.
|
|
143
|
+
Underpost.env.set('container-status', INIT_STATUS);
|
|
134
144
|
});
|
|
135
145
|
|
|
136
|
-
// Spawns the real monitorReadyRunner
|
|
137
|
-
//
|
|
138
|
-
const spawnMonitor = () =>
|
|
146
|
+
// Spawns the real monitorReadyRunner with the fake cluster on PATH; resolves
|
|
147
|
+
// with its exit code. `overrides` inject deterministic timing / target port.
|
|
148
|
+
const spawnMonitor = (overrides = {}) =>
|
|
139
149
|
new Promise((resolve) => {
|
|
140
|
-
const
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
150
|
+
const envVars = {
|
|
151
|
+
PATH: `${fakeBinDir}:${process.env.PATH}`,
|
|
152
|
+
UNDERPOST_ENV_FILE: envFile,
|
|
153
|
+
POD_NAME,
|
|
154
|
+
npm_config_prefix: tmpPrefix,
|
|
155
|
+
UNDERPOST_INTERNAL_PORT: String(INTERNAL_PORT),
|
|
156
|
+
// Pin the tunnel's local port so the no-op fake port-forward + the real
|
|
157
|
+
// internal server (bound to INTERNAL_PORT in this process) resolve to the
|
|
158
|
+
// same address the monitor's HTTP GET targets.
|
|
159
|
+
UNDERPOST_PF_LOCAL_PORT: String(INTERNAL_PORT),
|
|
160
|
+
UNDERPOST_MONITOR_DELAY_MS: '100',
|
|
161
|
+
UNDERPOST_MONITOR_MAX_ITERATIONS: '60',
|
|
162
|
+
UNDERPOST_PF_ATTEMPTS: '3',
|
|
163
|
+
FAKE_POD_READY: 'True',
|
|
164
|
+
...overrides,
|
|
165
|
+
};
|
|
166
|
+
const prefix = Object.entries(envVars)
|
|
167
|
+
.map(([k, v]) => `${k}="${v}"`)
|
|
168
|
+
.join(' ');
|
|
145
169
|
shellExec(`${prefix} ${node} ${monitorScriptPath}`, {
|
|
146
170
|
async: true,
|
|
147
171
|
silent: true,
|
|
@@ -150,39 +174,50 @@ try {
|
|
|
150
174
|
});
|
|
151
175
|
});
|
|
152
176
|
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
const deployCmd = `${node} -e "process.exit(${shouldFail ? 1 : 0})"`;
|
|
159
|
-
shellExec(deployCmd, {
|
|
160
|
-
async: true,
|
|
161
|
-
silent: true,
|
|
162
|
-
disableLog: true,
|
|
163
|
-
callback: (code) => {
|
|
164
|
-
if (code !== 0) Underpost.env.set('container-status', 'error');
|
|
165
|
-
else Underpost.env.set('container-status', RUNNING_STATUS);
|
|
166
|
-
resolve(code);
|
|
167
|
-
},
|
|
168
|
-
});
|
|
169
|
-
});
|
|
170
|
-
|
|
171
|
-
it('error: start.js sets container-status=error and monitorReadyRunner exits 1', async () => {
|
|
172
|
-
const monitorExit = spawnMonitor();
|
|
173
|
-
const deployCode = await runStartJs(true);
|
|
177
|
+
it('success: both phases satisfied → monitor exits 0', async () => {
|
|
178
|
+
Underpost.env.set('container-status', RUNNING_STATUS);
|
|
179
|
+
const code = await spawnMonitor({ FAKE_POD_READY: 'True' });
|
|
180
|
+
expect(code).to.equal(0);
|
|
181
|
+
});
|
|
174
182
|
|
|
175
|
-
|
|
176
|
-
|
|
183
|
+
it('runtime failure: container-status=error → monitor exits 1', async () => {
|
|
184
|
+
const monitorExit = spawnMonitor({ FAKE_POD_READY: 'True' });
|
|
185
|
+
Underpost.env.set('container-status', 'error');
|
|
177
186
|
expect(await monitorExit).to.equal(1);
|
|
178
187
|
});
|
|
179
188
|
|
|
180
|
-
it('
|
|
181
|
-
|
|
182
|
-
|
|
189
|
+
it('readiness mismatch: runtime running but pod not Ready → never succeeds (exits 1)', async () => {
|
|
190
|
+
// Phase 2 satisfied, Phase 1 not: success requires BOTH, so it must time out.
|
|
191
|
+
Underpost.env.set('container-status', RUNNING_STATUS);
|
|
192
|
+
const code = await spawnMonitor({ FAKE_POD_READY: 'False', UNDERPOST_MONITOR_MAX_ITERATIONS: '4' });
|
|
193
|
+
expect(code).to.equal(1);
|
|
194
|
+
});
|
|
183
195
|
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
196
|
+
it('transport failure: endpoint unreachable is never success (exits 1)', async () => {
|
|
197
|
+
// Point the monitor at a port with no internal server; the HTTP read always
|
|
198
|
+
// fails, so runtime readiness is never confirmed and the monitor times out.
|
|
199
|
+
Underpost.env.set('container-status', RUNNING_STATUS);
|
|
200
|
+
const code = await spawnMonitor({
|
|
201
|
+
UNDERPOST_INTERNAL_PORT: String(CLOSED_PORT),
|
|
202
|
+
UNDERPOST_PF_LOCAL_PORT: String(CLOSED_PORT),
|
|
203
|
+
UNDERPOST_MONITOR_MAX_ITERATIONS: '3',
|
|
204
|
+
});
|
|
205
|
+
expect(code).to.equal(1);
|
|
206
|
+
});
|
|
207
|
+
|
|
208
|
+
it('timeout: runtime stuck initializing → monitor exits 1', async () => {
|
|
209
|
+
Underpost.env.set('container-status', INIT_STATUS);
|
|
210
|
+
const code = await spawnMonitor({ FAKE_POD_READY: 'True', UNDERPOST_MONITOR_MAX_ITERATIONS: '4' });
|
|
211
|
+
expect(code).to.equal(1);
|
|
212
|
+
});
|
|
213
|
+
|
|
214
|
+
it('regression: advanced pod whose runtime status falls back → monitor exits 1', async () => {
|
|
215
|
+
// Pod advances past build, then its reported status regresses (pod restart);
|
|
216
|
+
// the monitor must treat this as a failure rather than wait it out.
|
|
217
|
+
Underpost.env.set('container-status', INIT_STATUS);
|
|
218
|
+
const monitorExit = spawnMonitor({ FAKE_POD_READY: 'False', UNDERPOST_MONITOR_MAX_ITERATIONS: '120' });
|
|
219
|
+
await new Promise((r) => setTimeout(r, 1500));
|
|
220
|
+
Underpost.env.set('container-status', BUILD_STATUS);
|
|
221
|
+
expect(await monitorExit).to.equal(1);
|
|
187
222
|
});
|
|
188
223
|
});
|