underpost 3.2.14 → 3.2.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +94 -1
- package/CLI-HELP.md +99 -30
- package/README.md +2 -2
- package/bin/build.js +12 -1
- package/bin/build.template.js +4 -2
- package/manifests/cronjobs/dd-cron/dd-cron-backup.yaml +1 -1
- package/manifests/cronjobs/dd-cron/dd-cron-dns.yaml +1 -1
- package/manifests/deployment/dd-default-development/deployment.yaml +2 -2
- package/package.json +1 -1
- package/scripts/link-local-underpost-cli.sh +6 -0
- package/scripts/test-monitor.sh +250 -0
- package/src/cli/deploy.js +200 -54
- package/src/cli/env.js +1 -4
- package/src/cli/index.js +47 -0
- package/src/cli/monitor.js +269 -72
- package/src/cli/release.js +21 -6
- package/src/cli/repository.js +21 -4
- package/src/cli/run.js +44 -4
- package/src/client/components/core/PanelForm.js +44 -44
- package/src/db/mongo/MongooseDB.js +2 -1
- package/src/index.js +1 -1
- package/src/server/conf.js +91 -18
- package/src/server/ipfs-client.js +5 -3
- package/src/server/runtime-status.js +235 -0
- package/src/server/start.js +26 -9
- package/test/deploy-monitor.test.js +132 -69
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Runtime status contract and the in-pod internal status endpoint.
|
|
3
|
+
*
|
|
4
|
+
* Single source of truth for the Underpost runtime readiness signal (Phase 2 of
|
|
5
|
+
* the two-phase deployment monitor). The runtime publishes its lifecycle here;
|
|
6
|
+
* the CD-side monitor (`src/cli/monitor.js`) reads it over HTTP via
|
|
7
|
+
* `kubectl port-forward`. Kubernetes pod readiness (Phase 1) is owned by kubelet
|
|
8
|
+
* and is intentionally not modeled in this module.
|
|
9
|
+
*
|
|
10
|
+
* Cross-process contract:
|
|
11
|
+
* - In-pod, the canonical value lives in the underpost root env key
|
|
12
|
+
* `container-status`, written by `start.js`. For non-error phases it carries
|
|
13
|
+
* the namespaced form `<deployId>-<env>-<phase>`; a fatal fault collapses to
|
|
14
|
+
* the bare value `error`.
|
|
15
|
+
* - The internal HTTP server exposes that value (normalized to the bare
|
|
16
|
+
* contract phase) and never exposes secrets, env dumps, or configuration.
|
|
17
|
+
*
|
|
18
|
+
* @module src/server/runtime-status.js
|
|
19
|
+
* @namespace RuntimeStatus
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import http from 'node:http';
|
|
23
|
+
import fs from 'fs-extra';
|
|
24
|
+
import dotenv from 'dotenv';
|
|
25
|
+
import Underpost from '../index.js';
|
|
26
|
+
import { loggerFactory } from './logger.js';
|
|
27
|
+
|
|
28
|
+
const logger = loggerFactory(import.meta);
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Allowed runtime status contract values. These are the only Phase-2 signals
|
|
32
|
+
* the monitor reasons about.
|
|
33
|
+
* @memberof RuntimeStatus
|
|
34
|
+
*/
|
|
35
|
+
const RUNTIME_STATUS = {
|
|
36
|
+
BUILD: 'build-deployment',
|
|
37
|
+
INIT: 'initializing-deployment',
|
|
38
|
+
RUNNING: 'running-deployment',
|
|
39
|
+
ERROR: 'error',
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
const CONTAINER_STATUS_KEY = 'container-status';
|
|
43
|
+
const INTERNAL_STATUS_PATH = '/_internal/status';
|
|
44
|
+
const INTERNAL_READY_PATH = '/_internal/ready';
|
|
45
|
+
const INTERNAL_HEALTH_PATH = '/_internal/health';
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Resolves the internal status port. Defaults to the deployment base `PORT`
|
|
49
|
+
* (app instances bind `PORT + 1` upward, so the base port is free inside the
|
|
50
|
+
* pod). An explicit `UNDERPOST_INTERNAL_PORT` override wins.
|
|
51
|
+
* @memberof RuntimeStatus
|
|
52
|
+
* @returns {number|undefined}
|
|
53
|
+
*/
|
|
54
|
+
const resolveInternalStatusPort = () => {
|
|
55
|
+
const raw = process.env.UNDERPOST_INTERNAL_PORT || process.env.PORT;
|
|
56
|
+
const port = parseInt(raw);
|
|
57
|
+
return Number.isNaN(port) ? undefined : port;
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Single source of truth for the internal status port of a specific deployment,
|
|
62
|
+
* used identically by the in-pod server bind (`start.js`) and the CD-side
|
|
63
|
+
* monitor target (`monitor.js`) so the two can never disagree.
|
|
64
|
+
*
|
|
65
|
+
* Resolution order: `UNDERPOST_INTERNAL_PORT` override → the deployment's
|
|
66
|
+
* `.env.<env>` `PORT` → the ambient `PORT`.
|
|
67
|
+
*
|
|
68
|
+
* @memberof RuntimeStatus
|
|
69
|
+
* @param {string} deployId
|
|
70
|
+
* @param {string} env
|
|
71
|
+
* @returns {number|undefined}
|
|
72
|
+
*/
|
|
73
|
+
const deployStatusPort = (deployId, env) => {
|
|
74
|
+
const override = parseInt(process.env.UNDERPOST_INTERNAL_PORT);
|
|
75
|
+
if (!Number.isNaN(override)) return override;
|
|
76
|
+
try {
|
|
77
|
+
const envPath = `./engine-private/conf/${deployId}/.env.${env}`;
|
|
78
|
+
if (fs.existsSync(envPath)) {
|
|
79
|
+
const port = parseInt(dotenv.parse(fs.readFileSync(envPath, 'utf8')).PORT);
|
|
80
|
+
if (!Number.isNaN(port)) return port;
|
|
81
|
+
}
|
|
82
|
+
} catch (_) {
|
|
83
|
+
/* fall through to ambient resolution */
|
|
84
|
+
}
|
|
85
|
+
return resolveInternalStatusPort();
|
|
86
|
+
};
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Builds the `container-status` env value for a lifecycle phase.
|
|
90
|
+
* @memberof RuntimeStatus
|
|
91
|
+
*/
|
|
92
|
+
const containerStatusValue = (deployId, env, phase) =>
|
|
93
|
+
phase === RUNTIME_STATUS.ERROR ? RUNTIME_STATUS.ERROR : `${deployId}-${env}-${phase}`;
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Normalizes a raw `container-status` value to a bare contract phase.
|
|
97
|
+
* Strips the `<deployId>-<env>-` prefix; `error` and unknown/empty values are
|
|
98
|
+
* passed through (empty → undefined).
|
|
99
|
+
* @memberof RuntimeStatus
|
|
100
|
+
* @param {string} raw
|
|
101
|
+
* @returns {string|undefined}
|
|
102
|
+
*/
|
|
103
|
+
const normalizeContainerStatus = (raw) => {
|
|
104
|
+
if (!raw || typeof raw !== 'string') return undefined;
|
|
105
|
+
const value = raw.trim();
|
|
106
|
+
if (!value || value === 'undefined' || value.toLowerCase().includes('empty')) return undefined;
|
|
107
|
+
if (value === RUNTIME_STATUS.ERROR) return RUNTIME_STATUS.ERROR;
|
|
108
|
+
for (const phase of [RUNTIME_STATUS.BUILD, RUNTIME_STATUS.INIT, RUNTIME_STATUS.RUNNING])
|
|
109
|
+
if (value.endsWith(`-${phase}`)) return phase;
|
|
110
|
+
return value;
|
|
111
|
+
};
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Reads the current normalized runtime status from the env file.
|
|
115
|
+
* @memberof RuntimeStatus
|
|
116
|
+
* @returns {string|undefined}
|
|
117
|
+
*/
|
|
118
|
+
const getRuntimeStatus = () =>
|
|
119
|
+
normalizeContainerStatus(Underpost.env.get(CONTAINER_STATUS_KEY, undefined, { disableLog: true }));
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Minimal, secret-free payload served by the internal status endpoint and used
|
|
123
|
+
* by the monitor for failure classification and observability.
|
|
124
|
+
* @memberof RuntimeStatus
|
|
125
|
+
* @returns {{status: (string|null), deployId: (string|null), env: (string|null)}}
|
|
126
|
+
*/
|
|
127
|
+
const runtimeStatusPayload = () => ({
|
|
128
|
+
status: getRuntimeStatus() ?? null,
|
|
129
|
+
deployId: process.env.DEPLOY_ID ?? null,
|
|
130
|
+
env: process.env.NODE_ENV ?? null,
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Emits a structured, secret-free deployment transition event.
|
|
135
|
+
* @memberof RuntimeStatus
|
|
136
|
+
*/
|
|
137
|
+
const emitRuntimeEvent = ({ deployId, env, phase }) => {
|
|
138
|
+
logger.info('runtime-status', {
|
|
139
|
+
deployId,
|
|
140
|
+
env,
|
|
141
|
+
phase: 'runtime',
|
|
142
|
+
status: phase,
|
|
143
|
+
timestamp: new Date().toISOString(),
|
|
144
|
+
});
|
|
145
|
+
};
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* Publishes a runtime lifecycle phase to the cross-process contract.
|
|
149
|
+
* @memberof RuntimeStatus
|
|
150
|
+
* @param {string} deployId
|
|
151
|
+
* @param {string} env
|
|
152
|
+
* @param {string} phase - One of {@link RUNTIME_STATUS}.
|
|
153
|
+
*/
|
|
154
|
+
const setRuntimeStatus = (deployId, env, phase) => {
|
|
155
|
+
Underpost.env.set(CONTAINER_STATUS_KEY, containerStatusValue(deployId, env, phase));
|
|
156
|
+
emitRuntimeEvent({ deployId, env, phase });
|
|
157
|
+
};
|
|
158
|
+
|
|
159
|
+
let internalServer;
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* Starts the in-pod internal status server. Idempotent: repeated calls return
|
|
163
|
+
* the already-listening server. Exposes only the three internal endpoints and
|
|
164
|
+
* never serves secrets or configuration.
|
|
165
|
+
*
|
|
166
|
+
* GET /_internal/status → 200, `{status, deployId, env}` (monitor transport)
|
|
167
|
+
* GET /_internal/ready → 200 iff running-deployment, else 503 (readinessProbe)
|
|
168
|
+
* GET /_internal/health → 200 while the process is alive (livenessProbe)
|
|
169
|
+
*
|
|
170
|
+
* @memberof RuntimeStatus
|
|
171
|
+
* @param {number} [port]
|
|
172
|
+
* @returns {import('node:http').Server|undefined}
|
|
173
|
+
*/
|
|
174
|
+
const startInternalStatusServer = (port = resolveInternalStatusPort()) => {
|
|
175
|
+
if (internalServer) return internalServer;
|
|
176
|
+
if (!port) {
|
|
177
|
+
logger.warn('Internal status server not started: no resolvable port');
|
|
178
|
+
return undefined;
|
|
179
|
+
}
|
|
180
|
+
const server = http.createServer((req, res) => {
|
|
181
|
+
const url = (req.url || '').split('?')[0];
|
|
182
|
+
const sendJson = (code, body) => {
|
|
183
|
+
res.writeHead(code, { 'Content-Type': 'application/json' });
|
|
184
|
+
res.end(JSON.stringify(body));
|
|
185
|
+
};
|
|
186
|
+
if (req.method !== 'GET') return sendJson(405, { error: 'method_not_allowed' });
|
|
187
|
+
switch (url) {
|
|
188
|
+
case INTERNAL_HEALTH_PATH:
|
|
189
|
+
return sendJson(200, { status: 'ok' });
|
|
190
|
+
case INTERNAL_READY_PATH:
|
|
191
|
+
return getRuntimeStatus() === RUNTIME_STATUS.RUNNING
|
|
192
|
+
? sendJson(200, { status: RUNTIME_STATUS.RUNNING })
|
|
193
|
+
: sendJson(503, { status: getRuntimeStatus() ?? null });
|
|
194
|
+
case INTERNAL_STATUS_PATH:
|
|
195
|
+
return sendJson(200, runtimeStatusPayload());
|
|
196
|
+
default:
|
|
197
|
+
return sendJson(404, { error: 'not_found' });
|
|
198
|
+
}
|
|
199
|
+
});
|
|
200
|
+
server.on('error', (error) => logger.error('internal status server error', error?.message ?? error));
|
|
201
|
+
server.listen(port, () => logger.info(`Internal status endpoint listening on :${port}${INTERNAL_STATUS_PATH}`));
|
|
202
|
+
internalServer = server;
|
|
203
|
+
return internalServer;
|
|
204
|
+
};
|
|
205
|
+
|
|
206
|
+
/**
|
|
207
|
+
* Stops the internal status server if running. Returns a promise that resolves
|
|
208
|
+
* once the listener is closed. Primarily a test/teardown hook.
|
|
209
|
+
* @memberof RuntimeStatus
|
|
210
|
+
* @returns {Promise<void>}
|
|
211
|
+
*/
|
|
212
|
+
const stopInternalStatusServer = () =>
|
|
213
|
+
new Promise((resolve) => {
|
|
214
|
+
if (!internalServer) return resolve();
|
|
215
|
+
const server = internalServer;
|
|
216
|
+
internalServer = undefined;
|
|
217
|
+
server.close(() => resolve());
|
|
218
|
+
});
|
|
219
|
+
|
|
220
|
+
export {
|
|
221
|
+
RUNTIME_STATUS,
|
|
222
|
+
CONTAINER_STATUS_KEY,
|
|
223
|
+
INTERNAL_STATUS_PATH,
|
|
224
|
+
INTERNAL_READY_PATH,
|
|
225
|
+
INTERNAL_HEALTH_PATH,
|
|
226
|
+
resolveInternalStatusPort,
|
|
227
|
+
deployStatusPort,
|
|
228
|
+
containerStatusValue,
|
|
229
|
+
normalizeContainerStatus,
|
|
230
|
+
getRuntimeStatus,
|
|
231
|
+
runtimeStatusPayload,
|
|
232
|
+
setRuntimeStatus,
|
|
233
|
+
startInternalStatusServer,
|
|
234
|
+
stopInternalStatusServer,
|
|
235
|
+
};
|
package/src/server/start.js
CHANGED
|
@@ -8,6 +8,7 @@ import fs from 'fs-extra';
|
|
|
8
8
|
import { awaitDeployMonitor } from './conf.js';
|
|
9
9
|
import { actionInitLog, loggerFactory } from './logger.js';
|
|
10
10
|
import { shellCd, shellExec } from './process.js';
|
|
11
|
+
import { RUNTIME_STATUS, setRuntimeStatus, startInternalStatusServer, deployStatusPort } from './runtime-status.js';
|
|
11
12
|
import Underpost from '../index.js';
|
|
12
13
|
const logger = loggerFactory(import.meta);
|
|
13
14
|
|
|
@@ -147,10 +148,20 @@ class UnderpostStartUp {
|
|
|
147
148
|
pullBundle: false,
|
|
148
149
|
},
|
|
149
150
|
) {
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
151
|
+
// Bring the internal status endpoint up first so Phase-2 readiness is
|
|
152
|
+
// observable through every lifecycle phase, including build and init. Bind
|
|
153
|
+
// the deployment-resolved port so it always matches the monitor's target.
|
|
154
|
+
startInternalStatusServer(deployStatusPort(deployId, env));
|
|
155
|
+
try {
|
|
156
|
+
setRuntimeStatus(deployId, env, RUNTIME_STATUS.BUILD);
|
|
157
|
+
if (options.build === true) await Underpost.start.build(deployId, env, options);
|
|
158
|
+
setRuntimeStatus(deployId, env, RUNTIME_STATUS.INIT);
|
|
159
|
+
if (options.run === true) await Underpost.start.run(deployId, env, options);
|
|
160
|
+
} catch (error) {
|
|
161
|
+
logger.error('Deployment build/init failed', { deployId, env, message: error?.message });
|
|
162
|
+
setRuntimeStatus(deployId, env, RUNTIME_STATUS.ERROR);
|
|
163
|
+
if (!Underpost.env.isInsideContainer()) throw error;
|
|
164
|
+
}
|
|
154
165
|
},
|
|
155
166
|
/**
|
|
156
167
|
* Run itc-scripts and builds client bundle.
|
|
@@ -162,6 +173,8 @@ class UnderpostStartUp {
|
|
|
162
173
|
* @param {boolean} options.skipFullBuild - Whether to skip building the full client bundle.
|
|
163
174
|
* @param {boolean} options.pullBundle - When true, download pre-built client bundle from Cloudinary via pull-bundle (must be pushed first with push-bundle).
|
|
164
175
|
* This flag is independent of skipFullBuild: it can be combined with skipFullBuild or used alone.
|
|
176
|
+
* @param {boolean} options.privateTestRepo - When true, clone `engine-test-<id>` (the private test source repo
|
|
177
|
+
* published by `node bin/build <deployId> --update-private`) instead of the production `engine-<id>` repo.
|
|
165
178
|
* @memberof UnderpostStartUp
|
|
166
179
|
*/
|
|
167
180
|
async build(
|
|
@@ -170,7 +183,11 @@ class UnderpostStartUp {
|
|
|
170
183
|
options = { underpostQuicklyInstall: false, skipPullBase: false, skipFullBuild: false, pullBundle: false },
|
|
171
184
|
) {
|
|
172
185
|
const buildBasePath = `/home/dd`;
|
|
173
|
-
|
|
186
|
+
// `--private-test-repo` clones the isolated test source repo published by
|
|
187
|
+
// `node bin/build <deployId> --update-private`, instead of the production one.
|
|
188
|
+
const repoName = options?.privateTestRepo
|
|
189
|
+
? `engine-test-${deployId.split('-')[1]}`
|
|
190
|
+
: `engine-${deployId.split('-')[1]}`;
|
|
174
191
|
if (!options.skipPullBase) {
|
|
175
192
|
shellExec(`cd ${buildBasePath} && underpost clone ${process.env.GITHUB_USERNAME}/${repoName}`);
|
|
176
193
|
shellExec(`mkdir -p ${buildBasePath}/engine`);
|
|
@@ -201,7 +218,7 @@ class UnderpostStartUp {
|
|
|
201
218
|
const makeDeployCallback = (cmd) => (code, out, msg) => {
|
|
202
219
|
if (code !== 0) {
|
|
203
220
|
logger.error(`Deployment process exited with code ${code}`, { cmd, msg });
|
|
204
|
-
|
|
221
|
+
setRuntimeStatus(deployId, env, RUNTIME_STATUS.ERROR);
|
|
205
222
|
}
|
|
206
223
|
};
|
|
207
224
|
if (fs.existsSync(`./engine-private/replica`)) {
|
|
@@ -213,7 +230,7 @@ class UnderpostStartUp {
|
|
|
213
230
|
shellExec(replicaCmd, { async: true, callback: makeDeployCallback(replicaCmd) });
|
|
214
231
|
const result = await awaitDeployMonitor();
|
|
215
232
|
if (result !== true) {
|
|
216
|
-
|
|
233
|
+
setRuntimeStatus(deployId, env, RUNTIME_STATUS.ERROR);
|
|
217
234
|
return;
|
|
218
235
|
}
|
|
219
236
|
}
|
|
@@ -224,9 +241,9 @@ class UnderpostStartUp {
|
|
|
224
241
|
const result = await awaitDeployMonitor(true);
|
|
225
242
|
if (result === true) {
|
|
226
243
|
if (env === 'production' && Underpost.env.isInsideContainer()) Underpost.secret.globalSecretClean();
|
|
227
|
-
|
|
244
|
+
setRuntimeStatus(deployId, env, RUNTIME_STATUS.RUNNING);
|
|
228
245
|
} else {
|
|
229
|
-
|
|
246
|
+
setRuntimeStatus(deployId, env, RUNTIME_STATUS.ERROR);
|
|
230
247
|
}
|
|
231
248
|
},
|
|
232
249
|
};
|
|
@@ -1,28 +1,32 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @module deploy-monitor.test
|
|
3
|
-
* @description End-to-end test of the
|
|
4
|
-
* the in-pod runtime (start.js) and the CD-runner
|
|
5
|
-
* (`Underpost.monitor.monitorReadyRunner`), exercised as
|
|
6
|
-
*
|
|
3
|
+
* @description End-to-end test of the two-phase deployment readiness contract
|
|
4
|
+
* between the in-pod runtime (`start.js` / `runtime-status.js`) and the CD-runner
|
|
5
|
+
* monitor (`Underpost.monitor.monitorReadyRunner`), exercised as real OS
|
|
6
|
+
* processes coordinating through the shared underpost env file and a real HTTP
|
|
7
|
+
* internal status endpoint.
|
|
7
8
|
*
|
|
8
9
|
* Contract under test:
|
|
9
10
|
*
|
|
10
|
-
* -
|
|
11
|
-
*
|
|
12
|
-
* healthy one it sets `container-status=<deploy>-<env>-running-deployment`.
|
|
11
|
+
* - The in-pod runtime publishes only `container-status` (Phase 2) and serves
|
|
12
|
+
* it over `GET /_internal/status`. It never propagates an exit code.
|
|
13
13
|
*
|
|
14
|
-
* - monitorReadyRunner (CD runner)
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
* `
|
|
14
|
+
* - monitorReadyRunner (CD runner) confirms BOTH phases before exiting 0:
|
|
15
|
+
* Phase 1 — Kubernetes pod `Ready` condition (kubectl get pod -o json).
|
|
16
|
+
* Phase 2 — runtime `running-deployment` read over HTTP via
|
|
17
|
+
* `kubectl port-forward` to the internal endpoint.
|
|
18
|
+
* It throws (→ exit 1) on explicit runtime `error`, and returns (→ exit 0)
|
|
19
|
+
* only once both phases are satisfied.
|
|
19
20
|
*
|
|
20
|
-
* The cluster surface
|
|
21
|
-
* `
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
* the
|
|
21
|
+
* The cluster surface is supplied by fake `sudo`/`kubectl` binaries on PATH:
|
|
22
|
+
* - `get pods` / `get pod -o json` report one pod whose Ready condition is
|
|
23
|
+
* driven by FAKE_POD_READY.
|
|
24
|
+
* - `port-forward` is a no-op sleep; the monitor's HTTP GET reaches a REAL
|
|
25
|
+
* internal status server bound in this test process (localPort == port),
|
|
26
|
+
* which reads the same env file the runtime writes.
|
|
27
|
+
*
|
|
28
|
+
* The env file is redirected under a temp `npm_config_prefix`, so the test
|
|
29
|
+
* needs no cluster, no root, and never touches the machine's global install.
|
|
26
30
|
*
|
|
27
31
|
* Uses 'chai' for assertions.
|
|
28
32
|
*/
|
|
@@ -34,6 +38,7 @@ import path from 'node:path';
|
|
|
34
38
|
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
35
39
|
import Underpost from '../src/index.js';
|
|
36
40
|
import { shellExec } from '../src/server/process.js';
|
|
41
|
+
import { startInternalStatusServer, stopInternalStatusServer } from '../src/server/runtime-status.js';
|
|
37
42
|
|
|
38
43
|
const node = process.execPath;
|
|
39
44
|
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
|
@@ -43,9 +48,14 @@ const ENV = 'production';
|
|
|
43
48
|
const TRAFFIC = 'green';
|
|
44
49
|
const POD_NAME = `${DEPLOY_ID}-${ENV}-${TRAFFIC}-pod`;
|
|
45
50
|
const RUNNING_STATUS = `${DEPLOY_ID}-${ENV}-running-deployment`;
|
|
51
|
+
const INIT_STATUS = `${DEPLOY_ID}-${ENV}-initializing-deployment`;
|
|
52
|
+
const BUILD_STATUS = `${DEPLOY_ID}-${ENV}-build-deployment`;
|
|
53
|
+
|
|
54
|
+
const INTERNAL_PORT = 39517; // internal status endpoint (real server bound here)
|
|
55
|
+
const CLOSED_PORT = 39518; // no server — used to force a transport failure
|
|
46
56
|
|
|
47
|
-
describe('Deploy monitor —
|
|
48
|
-
this.timeout(
|
|
57
|
+
describe('Deploy monitor — two-phase state machine (e2e, real HTTP transport)', function () {
|
|
58
|
+
this.timeout(60000);
|
|
49
59
|
|
|
50
60
|
let prevPrefix;
|
|
51
61
|
let tmpPrefix;
|
|
@@ -58,15 +68,16 @@ describe('Deploy monitor — start.js ↔ monitorReadyRunner (e2e, parallel proc
|
|
|
58
68
|
tmpPrefix = fs.mkdtempSync(path.join(os.tmpdir(), 'underpost-e2e-'));
|
|
59
69
|
process.env.npm_config_prefix = tmpPrefix;
|
|
60
70
|
|
|
61
|
-
// Materialize the underpost env file and resolve its absolute path (the
|
|
62
|
-
// fake `kubectl exec` reads container-status straight from it).
|
|
63
71
|
Underpost.env.set('container-status', 'init');
|
|
64
72
|
const npmRoot = shellExec('npm root -g', { stdout: true, silent: true, disableLog: true }).trim();
|
|
65
73
|
envFile = path.join(npmRoot, 'underpost', '.env');
|
|
66
74
|
|
|
67
|
-
//
|
|
68
|
-
//
|
|
69
|
-
//
|
|
75
|
+
// Real in-pod internal status server: serves container-status from the same
|
|
76
|
+
// env file the runtime writes. Bound in this test process; the monitor's
|
|
77
|
+
// port-forward tunnel (localPort == INTERNAL_PORT) resolves straight to it.
|
|
78
|
+
process.env.UNDERPOST_INTERNAL_PORT = String(INTERNAL_PORT);
|
|
79
|
+
startInternalStatusServer(INTERNAL_PORT);
|
|
80
|
+
|
|
70
81
|
fakeBinDir = path.join(tmpPrefix, 'fakebin');
|
|
71
82
|
fs.ensureDirSync(fakeBinDir);
|
|
72
83
|
|
|
@@ -90,7 +101,11 @@ if [[ "$verb" == "get" && "$kind" == "pods" ]]; then
|
|
|
90
101
|
exit 0
|
|
91
102
|
fi
|
|
92
103
|
if [[ "$verb" == "get" && "$kind" == "pod" ]]; then
|
|
93
|
-
printf '{"status":{"conditions":[{"type":"Ready","status":"
|
|
104
|
+
printf '{"status":{"conditions":[{"type":"Ready","status":"%s"}]}}\\n' "\${FAKE_POD_READY:-True}"
|
|
105
|
+
exit 0
|
|
106
|
+
fi
|
|
107
|
+
if [[ "$verb" == "port-forward" ]]; then
|
|
108
|
+
sleep 30
|
|
94
109
|
exit 0
|
|
95
110
|
fi
|
|
96
111
|
if [[ "$verb" == "exec" ]]; then
|
|
@@ -103,17 +118,17 @@ exit 0
|
|
|
103
118
|
fs.chmodSync(sudoPath, 0o755);
|
|
104
119
|
fs.chmodSync(kubectlPath, 0o755);
|
|
105
120
|
|
|
106
|
-
// Real monitorReadyRunner in its own process: exit 0 when it returns (ready),
|
|
107
|
-
// exit 1 when it throws (container-status=error). This is the signal `set -e`
|
|
108
|
-
// turns into a passed/failed GitHub Actions job.
|
|
109
121
|
monitorScriptPath = path.join(tmpPrefix, 'monitor-ready.mjs');
|
|
110
122
|
fs.writeFileSync(
|
|
111
123
|
monitorScriptPath,
|
|
112
124
|
`import Underpost from ${JSON.stringify(pathToFileURL(path.join(repoRoot, 'src/index.js')).href)};
|
|
125
|
+
const options = {};
|
|
126
|
+
if (process.env.MON_READY_GATE) options.readyGate = process.env.MON_READY_GATE;
|
|
127
|
+
if (process.env.MON_TRANSPORT) options.statusTransport = process.env.MON_TRANSPORT;
|
|
113
128
|
try {
|
|
114
129
|
await Underpost.monitor.monitorReadyRunner(${JSON.stringify(DEPLOY_ID)}, ${JSON.stringify(ENV)}, ${JSON.stringify(
|
|
115
|
-
|
|
116
|
-
|
|
130
|
+
TRAFFIC,
|
|
131
|
+
)}, [], 'default', options);
|
|
117
132
|
process.exit(0);
|
|
118
133
|
} catch (_) {
|
|
119
134
|
process.exit(1);
|
|
@@ -122,26 +137,42 @@ try {
|
|
|
122
137
|
);
|
|
123
138
|
});
|
|
124
139
|
|
|
125
|
-
after(() => {
|
|
140
|
+
after(async () => {
|
|
141
|
+
await stopInternalStatusServer();
|
|
142
|
+
delete process.env.UNDERPOST_INTERNAL_PORT;
|
|
126
143
|
if (prevPrefix === undefined) delete process.env.npm_config_prefix;
|
|
127
144
|
else process.env.npm_config_prefix = prevPrefix;
|
|
128
145
|
fs.removeSync(tmpPrefix);
|
|
129
146
|
});
|
|
130
147
|
|
|
131
148
|
beforeEach(() => {
|
|
132
|
-
// Deploy in flight:
|
|
133
|
-
Underpost.env.set('container-status',
|
|
149
|
+
// Deploy in flight: app not yet reporting running.
|
|
150
|
+
Underpost.env.set('container-status', INIT_STATUS);
|
|
134
151
|
});
|
|
135
152
|
|
|
136
|
-
// Spawns the real monitorReadyRunner
|
|
137
|
-
//
|
|
138
|
-
const spawnMonitor = () =>
|
|
153
|
+
// Spawns the real monitorReadyRunner with the fake cluster on PATH; resolves
|
|
154
|
+
// with its exit code. `overrides` inject deterministic timing / target port.
|
|
155
|
+
const spawnMonitor = (overrides = {}) =>
|
|
139
156
|
new Promise((resolve) => {
|
|
140
|
-
const
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
157
|
+
const envVars = {
|
|
158
|
+
PATH: `${fakeBinDir}:${process.env.PATH}`,
|
|
159
|
+
UNDERPOST_ENV_FILE: envFile,
|
|
160
|
+
POD_NAME,
|
|
161
|
+
npm_config_prefix: tmpPrefix,
|
|
162
|
+
UNDERPOST_INTERNAL_PORT: String(INTERNAL_PORT),
|
|
163
|
+
// Pin the tunnel's local port so the no-op fake port-forward + the real
|
|
164
|
+
// internal server (bound to INTERNAL_PORT in this process) resolve to the
|
|
165
|
+
// same address the monitor's HTTP GET targets.
|
|
166
|
+
UNDERPOST_PF_LOCAL_PORT: String(INTERNAL_PORT),
|
|
167
|
+
UNDERPOST_MONITOR_DELAY_MS: '100',
|
|
168
|
+
UNDERPOST_MONITOR_MAX_ITERATIONS: '60',
|
|
169
|
+
UNDERPOST_PF_ATTEMPTS: '3',
|
|
170
|
+
FAKE_POD_READY: 'True',
|
|
171
|
+
...overrides,
|
|
172
|
+
};
|
|
173
|
+
const prefix = Object.entries(envVars)
|
|
174
|
+
.map(([k, v]) => `${k}="${v}"`)
|
|
175
|
+
.join(' ');
|
|
145
176
|
shellExec(`${prefix} ${node} ${monitorScriptPath}`, {
|
|
146
177
|
async: true,
|
|
147
178
|
silent: true,
|
|
@@ -150,39 +181,71 @@ try {
|
|
|
150
181
|
});
|
|
151
182
|
});
|
|
152
183
|
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
184
|
+
it('success (default exec transport): both phases satisfied → monitor exits 0', async () => {
|
|
185
|
+
Underpost.env.set('container-status', RUNNING_STATUS);
|
|
186
|
+
const code = await spawnMonitor({ FAKE_POD_READY: 'True' });
|
|
187
|
+
expect(code).to.equal(0);
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
it('success (opt-in http transport): both phases satisfied → monitor exits 0', async () => {
|
|
191
|
+
Underpost.env.set('container-status', RUNNING_STATUS);
|
|
192
|
+
const code = await spawnMonitor({ FAKE_POD_READY: 'True', MON_TRANSPORT: 'http' });
|
|
193
|
+
expect(code).to.equal(0);
|
|
194
|
+
});
|
|
195
|
+
|
|
196
|
+
it('runtime failure: container-status=error → monitor exits 1', async () => {
|
|
197
|
+
const monitorExit = spawnMonitor({ FAKE_POD_READY: 'True' });
|
|
198
|
+
Underpost.env.set('container-status', 'error');
|
|
199
|
+
expect(await monitorExit).to.equal(1);
|
|
200
|
+
});
|
|
201
|
+
|
|
202
|
+
it('readiness mismatch: runtime running but pod not Ready → never succeeds (exits 1)', async () => {
|
|
203
|
+
// Phase 2 satisfied, Phase 1 not: success requires BOTH, so it must time out.
|
|
204
|
+
Underpost.env.set('container-status', RUNNING_STATUS);
|
|
205
|
+
const code = await spawnMonitor({ FAKE_POD_READY: 'False', UNDERPOST_MONITOR_MAX_ITERATIONS: '4' });
|
|
206
|
+
expect(code).to.equal(1);
|
|
207
|
+
});
|
|
208
|
+
|
|
209
|
+
it('transport failure (http): endpoint unreachable is never success (exits 1)', async () => {
|
|
210
|
+
// Opt into http and point the monitor at a port with no internal server; the
|
|
211
|
+
// HTTP read always fails, so runtime readiness is never confirmed → timeout.
|
|
212
|
+
Underpost.env.set('container-status', RUNNING_STATUS);
|
|
213
|
+
const code = await spawnMonitor({
|
|
214
|
+
MON_TRANSPORT: 'http',
|
|
215
|
+
UNDERPOST_INTERNAL_PORT: String(CLOSED_PORT),
|
|
216
|
+
UNDERPOST_PF_LOCAL_PORT: String(CLOSED_PORT),
|
|
217
|
+
UNDERPOST_MONITOR_MAX_ITERATIONS: '3',
|
|
169
218
|
});
|
|
219
|
+
expect(code).to.equal(1);
|
|
220
|
+
});
|
|
170
221
|
|
|
171
|
-
it('
|
|
172
|
-
|
|
173
|
-
const
|
|
222
|
+
it('timeout: runtime stuck initializing → monitor exits 1', async () => {
|
|
223
|
+
Underpost.env.set('container-status', INIT_STATUS);
|
|
224
|
+
const code = await spawnMonitor({ FAKE_POD_READY: 'True', UNDERPOST_MONITOR_MAX_ITERATIONS: '4' });
|
|
225
|
+
expect(code).to.equal(1);
|
|
226
|
+
});
|
|
174
227
|
|
|
175
|
-
|
|
176
|
-
|
|
228
|
+
it('regression: advanced pod whose runtime status falls back → monitor exits 1', async () => {
|
|
229
|
+
// Pod advances past build, then its reported status regresses (pod restart);
|
|
230
|
+
// the monitor must treat this as a failure rather than wait it out.
|
|
231
|
+
Underpost.env.set('container-status', INIT_STATUS);
|
|
232
|
+
const monitorExit = spawnMonitor({ FAKE_POD_READY: 'False', UNDERPOST_MONITOR_MAX_ITERATIONS: '120' });
|
|
233
|
+
await new Promise((r) => setTimeout(r, 1500));
|
|
234
|
+
Underpost.env.set('container-status', BUILD_STATUS);
|
|
177
235
|
expect(await monitorExit).to.equal(1);
|
|
178
236
|
});
|
|
179
237
|
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
238
|
+
// Custom instances (cyberia-*) gate on K8s Ready and read status via exec;
|
|
239
|
+
// their runtime never stamps `running-deployment` (stays `initializing`).
|
|
240
|
+
it('instance (kubernetes gate + exec): K8s Ready with initializing status → exits 0', async () => {
|
|
241
|
+
Underpost.env.set('container-status', INIT_STATUS);
|
|
242
|
+
const code = await spawnMonitor({ FAKE_POD_READY: 'True', MON_READY_GATE: 'kubernetes', MON_TRANSPORT: 'exec' });
|
|
243
|
+
expect(code).to.equal(0);
|
|
244
|
+
});
|
|
183
245
|
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
246
|
+
it('instance (kubernetes gate + exec): container-status=error → exits 1', async () => {
|
|
247
|
+
const monitorExit = spawnMonitor({ FAKE_POD_READY: 'True', MON_READY_GATE: 'kubernetes', MON_TRANSPORT: 'exec' });
|
|
248
|
+
Underpost.env.set('container-status', 'error');
|
|
249
|
+
expect(await monitorExit).to.equal(1);
|
|
187
250
|
});
|
|
188
251
|
});
|