rollbridge 0.1.73 → 0.1.75
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +63 -10
- package/TODO.md +13 -0
- package/changelog.d/20261025-guardian-re-register-dead-generation.md +13 -0
- package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +1 -0
- package/changelog.d/20261104-compose-service-units.md +21 -0
- package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +20 -0
- package/changelog.d/20261106-release-dependency-boot-budgets.md +32 -0
- package/changelog.d/20261109-path-filtered-builds.md +11 -0
- package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +26 -0
- package/changelog.d/20261111-contention-sensitive-daemon-specs.md +27 -0
- package/changelog.d/20261113-bounded-nonblocking-retirement.md +5 -0
- package/changelog.d/20261115-bound-draining-release-groups.md +62 -0
- package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +27 -0
- package/changelog.d/20261115-wedged-generation-escalation.md +44 -0
- package/changelog.d/20261116-bounded-explicit-stop-retirement.md +30 -0
- package/changelog.d/20261116-config-json-canonical-shape.md +11 -0
- package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +29 -0
- package/changelog.d/20261116-guardian-version-handoff.md +37 -0
- package/changelog.d/20261116-reclaim-dead-owner-mutation.md +7 -0
- package/changelog.d/20261117-activate-before-incumbent-retirement.md +1 -0
- package/changelog.d/20261117-activation-path-drain-deadline.md +35 -0
- package/changelog.d/20261117-stop-companion-after-retired-dependency.md +7 -0
- package/changelog.d/20261118-bounded-control-client-close.md +4 -0
- package/changelog.d/20261118-incumbent-retirement-authority-guard.md +1 -0
- package/changelog.d/20261118-owner-retirement-no-guardian-bound.md +19 -0
- package/changelog.d/20261118-shutdown-retirement-bound-restored.md +31 -0
- package/docs/cli.md +128 -3
- package/docs/compose-services.md +222 -0
- package/docs/config.md +177 -6
- package/docs/troubleshooting.md +97 -6
- package/docs/velocious.md +1 -1
- package/docs/workers.md +11 -0
- package/examples/tensorbuzz.com.js +3 -2
- package/package.json +1 -1
- package/src/cli.js +69 -3
- package/src/compose-unit.js +725 -0
- package/src/config.js +346 -8
- package/src/control-client.js +32 -5
- package/src/daemon.js +448 -76
- package/src/doctor.js +74 -0
- package/src/drain-policy.js +252 -0
- package/src/guardian-client.js +92 -9
- package/src/health.js +99 -8
- package/src/managed-process.js +197 -16
- package/src/process-guardian.js +212 -22
- package/src/process-memory.js +62 -2
- package/src/process-order.js +26 -0
- package/src/release-group.js +452 -39
- package/tensorbuzz.yml +20 -1
- package/test/completion.test.js +5 -2
- package/test/compose-config.test.js +160 -0
- package/test/compose-docker.test.js +297 -0
- package/test/compose-unit.test.js +415 -0
- package/test/config-validation.test.js +135 -5
- package/test/control-protocol.test.js +22 -0
- package/test/drain-policy.test.js +328 -0
- package/test/fixtures/deferred-boot-app.js +106 -0
- package/test/fixtures/gated-app.js +50 -0
- package/test/guardian-client.test.js +149 -5
- package/test/health.test.js +74 -1
- package/test/managed-process.test.js +260 -1
- package/test/owner-recovery.test.js +151 -20
- package/test/owner-replacement.test.js +176 -15
- package/test/process-memory.test.js +47 -1
- package/test/release-group.test.js +447 -2
- package/test/rollbridge.test.js +774 -72
- package/test/shutdown-completion.test.js +119 -0
- package/test/shutdown-retirement-bound.test.js +162 -0
- package/test/support/process.js +3 -1
- package/test/tensorbuzz-config.test.js +228 -0
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
3
|
import {describe, expect, test} from "@velocious/testing"
|
|
4
|
+
import fs from "node:fs"
|
|
5
|
+
import os from "node:os"
|
|
6
|
+
import path from "node:path"
|
|
4
7
|
import ReleaseGroup from "../src/release-group.js"
|
|
5
8
|
import {normalizeConfig} from "../src/config.js"
|
|
6
9
|
|
|
@@ -9,9 +12,10 @@ describe("release-group", () => {
|
|
|
9
12
|
/**
|
|
10
13
|
* @param {import("../src/json.js").JsonValue} webProcess - The single proxied process definition.
|
|
11
14
|
* @param {() => boolean} [shouldStart] - Whether process starts remain allowed.
|
|
15
|
+
* @param {string} [procPath] - Procfs root for drain liveness observations.
|
|
12
16
|
* @returns {ReleaseGroup} A release group ready for buildProcess.
|
|
13
17
|
*/
|
|
14
|
-
function buildRelease(webProcess, shouldStart = () => true) {
|
|
18
|
+
function buildRelease(webProcess, shouldStart = () => true, procPath = "/proc") {
|
|
15
19
|
const config = normalizeConfig({
|
|
16
20
|
application: "demo",
|
|
17
21
|
control: {path: "/tmp/rollbridge-release-group.sock"},
|
|
@@ -19,7 +23,37 @@ function buildRelease(webProcess, shouldStart = () => true) {
|
|
|
19
23
|
proxy: {host: "127.0.0.1", port: 0}
|
|
20
24
|
})
|
|
21
25
|
|
|
22
|
-
return new ReleaseGroup({config, logger: () => {}, releaseId: "v1", releasePath: "/tmp/rel", revision: "v1", shouldStart})
|
|
26
|
+
return new ReleaseGroup({config, logger: () => {}, procPath, releaseId: "v1", releasePath: "/tmp/rel", revision: "v1", shouldStart})
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* @param {string} procPath - Procfs root.
|
|
31
|
+
* @param {number} pid - Process id.
|
|
32
|
+
* @param {number} pgrp - Process group id.
|
|
33
|
+
* @param {string} state - Process state letter.
|
|
34
|
+
* @param {number} utimeTicks - Cumulative user CPU in clock ticks.
|
|
35
|
+
* @param {number} stimeTicks - Cumulative system CPU in clock ticks.
|
|
36
|
+
* @returns {void} One synthetic procfs member.
|
|
37
|
+
*/
|
|
38
|
+
function writeProcMember(procPath, pid, pgrp, state, utimeTicks, stimeTicks) {
|
|
39
|
+
fs.mkdirSync(path.join(procPath, String(pid)), {recursive: true})
|
|
40
|
+
|
|
41
|
+
// Eight placeholder fields sit between pgrp and utime, which is stat field 14 (index 11
|
|
42
|
+
// after the parenthesised comm); stime is field 15 (index 12).
|
|
43
|
+
fs.writeFileSync(path.join(procPath, String(pid), "stat"), `${pid} (proc) ${state} 1 ${pgrp} 0 0 0 0 0 0 0 0 ${utimeTicks} ${stimeTicks}`)
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* @param {number | undefined} pid - Live pid, or undefined when the tracked process is gone.
|
|
48
|
+
* @param {import("../src/managed-process.js").HeldProcessGroup | undefined} [heldProcessGroup] - Group still alive after the tracked process exited.
|
|
49
|
+
* @returns {import("../src/managed-process.js").default} A process whose record is already terminal.
|
|
50
|
+
*/
|
|
51
|
+
function drainedProcess(pid, heldProcessGroup = undefined) {
|
|
52
|
+
// eslint-disable-next-line local/no-unknown-jsdoc-type
|
|
53
|
+
return /** @type {import("../src/managed-process.js").default} */ (/** @type {unknown} */ ({
|
|
54
|
+
retirementTimedOut: true,
|
|
55
|
+
status: () => ({heldProcessGroup, pid, retirementTimedOut: true, state: "stopped"})
|
|
56
|
+
}))
|
|
23
57
|
}
|
|
24
58
|
|
|
25
59
|
test("templates interpolate values from the daemon environment", () => {
|
|
@@ -65,6 +99,42 @@ test("replica processes get a replica index, count, and template context", () =>
|
|
|
65
99
|
expect(replica.env.SLOT).toBe("1")
|
|
66
100
|
})
|
|
67
101
|
|
|
102
|
+
test("a companion stops without restarting after its same-generation service dependency retires", async () => {
|
|
103
|
+
const config = normalizeConfig({
|
|
104
|
+
application: "demo",
|
|
105
|
+
control: {path: "/tmp/rollbridge-release-group.sock"},
|
|
106
|
+
processes: [
|
|
107
|
+
{
|
|
108
|
+
command: "run jobs-main",
|
|
109
|
+
id: "jobs-main",
|
|
110
|
+
lifecycle: {quietCommand: "retire jobs-main"},
|
|
111
|
+
policy: "service",
|
|
112
|
+
port: {from: 7331, to: 7399}
|
|
113
|
+
},
|
|
114
|
+
{command: "run worker", dependsOn: ["jobs-main"], id: "worker", policy: "companion"},
|
|
115
|
+
{command: "run web", id: "web", policy: "proxied", port: {from: 14500, to: 14599}}
|
|
116
|
+
],
|
|
117
|
+
proxy: {host: "127.0.0.1", port: 0}
|
|
118
|
+
})
|
|
119
|
+
const release = new ReleaseGroup({config, logger: () => {}, releaseId: "v1", releasePath: "/tmp/rel", revision: "v1"})
|
|
120
|
+
const jobsMain = release.buildProcess(config.processes[0])
|
|
121
|
+
const worker = release.buildProcess(config.processes[1])
|
|
122
|
+
|
|
123
|
+
release.state = "active"
|
|
124
|
+
release.processes.set("jobs-main", jobsMain)
|
|
125
|
+
release.processes.set("worker", worker)
|
|
126
|
+
|
|
127
|
+
await jobsMain.setLifecycleRole("retired")
|
|
128
|
+
worker.command = `${JSON.stringify(process.execPath)} -e ${JSON.stringify("process.exit(1)")}`
|
|
129
|
+
worker.cwd = undefined
|
|
130
|
+
await new Promise((resolve, reject) => {
|
|
131
|
+
worker.once("exit", resolve)
|
|
132
|
+
worker.start("deploy").catch(reject)
|
|
133
|
+
})
|
|
134
|
+
|
|
135
|
+
expect(worker.status()).toMatchObject({restarts: 0, state: "stopped"})
|
|
136
|
+
})
|
|
137
|
+
|
|
68
138
|
test("a referenced daemon environment variable that is unset fails fast", async () => {
|
|
69
139
|
const release = buildRelease({
|
|
70
140
|
command: "run {{env.ROLLBRIDGE_ENV_MISSING}}",
|
|
@@ -109,6 +179,96 @@ test("stopping a release abandons its connection count instead of persisting a s
|
|
|
109
179
|
expect(release.localConnections()).toEqual({http: 0, websocket: 0})
|
|
110
180
|
})
|
|
111
181
|
|
|
182
|
+
test("a wedged non-blocking process cannot block release retirement", async () => {
|
|
183
|
+
/** @type {Array<{message: string, data?: import("../src/json.js").JsonValue}>} */
|
|
184
|
+
const logs = []
|
|
185
|
+
const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}})
|
|
186
|
+
release.logger = (message, data) => logs.push({message, data})
|
|
187
|
+
release.state = "draining"
|
|
188
|
+
release.nonBlockingDrainIds.add("worker")
|
|
189
|
+
release.handoffServiceIds.add("jobs-main")
|
|
190
|
+
// eslint-disable-next-line local/no-unknown-jsdoc-type
|
|
191
|
+
release.processes.set("worker", /** @type {import("../src/managed-process.js").default} */ (/** @type {unknown} */ ({
|
|
192
|
+
quiesceStrict: async () => {},
|
|
193
|
+
status: () => ({pid: 4242}),
|
|
194
|
+
retirementTimedOut: true,
|
|
195
|
+
stop: async () => {}
|
|
196
|
+
})))
|
|
197
|
+
let handoffStopped = false
|
|
198
|
+
// eslint-disable-next-line local/no-unknown-jsdoc-type
|
|
199
|
+
release.processes.set("jobs-main", /** @type {import("../src/managed-process.js").default} */ (/** @type {unknown} */ ({
|
|
200
|
+
stop: async () => { handoffStopped = true }
|
|
201
|
+
})))
|
|
202
|
+
|
|
203
|
+
await release.drainAndStop(0, {...release.config, proxy: {...release.config.proxy, nonBlockingDrainTimeoutMs: 20}})
|
|
204
|
+
|
|
205
|
+
expect(release.state).toBe("stopped")
|
|
206
|
+
expect(handoffStopped).toBe(true)
|
|
207
|
+
expect(logs).toMatchObject([{message: "release non-blocking process still draining", data: {pid: 4242, processId: "worker"}}])
|
|
208
|
+
})
|
|
209
|
+
|
|
210
|
+
test("an explicit stop bounds the processes it must not kill and reports what it left alive", async () => {
|
|
211
|
+
/** @type {Array<{message: string, data?: import("../src/json.js").JsonValue}>} */
|
|
212
|
+
const logs = []
|
|
213
|
+
/** @type {Array<{id: string, options: {retirementTimeoutMs?: number, timeoutMs?: number}}>} */
|
|
214
|
+
const stopOptions = []
|
|
215
|
+
const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}})
|
|
216
|
+
|
|
217
|
+
release.logger = (message, data) => logs.push({message, data})
|
|
218
|
+
release.nonBlockingDrainIds.add("worker")
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* @param {string} id - Process id.
|
|
222
|
+
* @param {boolean} unboundedWindow - Whether its graceful window can ever end.
|
|
223
|
+
* @returns {void} A process recorded in the generation.
|
|
224
|
+
*/
|
|
225
|
+
const addProcess = (id, unboundedWindow) => {
|
|
226
|
+
const instance = {
|
|
227
|
+
hasUnboundedGracefulWindow: () => unboundedWindow,
|
|
228
|
+
retirementTimedOut: false,
|
|
229
|
+
status: () => ({pid: 4242, state: "stopped"}),
|
|
230
|
+
stop: async (/** @type {{retirementTimeoutMs?: number, timeoutMs?: number}} */ options) => {
|
|
231
|
+
stopOptions.push({id, options})
|
|
232
|
+
// Only the retirement branch sets this, so a process that was never budgeted never claims it.
|
|
233
|
+
if (options.retirementTimeoutMs !== undefined) instance.retirementTimedOut = true
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
// eslint-disable-next-line local/no-unknown-jsdoc-type
|
|
238
|
+
release.processes.set(id, /** @type {import("../src/managed-process.js").default} */ (/** @type {unknown} */ (instance)))
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
addProcess("worker", false)
|
|
242
|
+
addProcess("jobs-main", true)
|
|
243
|
+
addProcess("web", false)
|
|
244
|
+
|
|
245
|
+
await release.stop({retirementTimeoutMs: 40})
|
|
246
|
+
|
|
247
|
+
expect(stopOptions).toEqual([
|
|
248
|
+
{id: "worker", options: {retirementTimeoutMs: 40}},
|
|
249
|
+
{id: "jobs-main", options: {retirementTimeoutMs: 40}},
|
|
250
|
+
{id: "web", options: {}}
|
|
251
|
+
])
|
|
252
|
+
expect(release.state).toBe("stopped")
|
|
253
|
+
expect(logs).toMatchObject([
|
|
254
|
+
{message: "release stop gave up waiting for a retiring process", data: {pid: 4242, processId: "worker", releaseId: "v1", timeoutMs: 40}},
|
|
255
|
+
{message: "release stop gave up waiting for a retiring process", data: {pid: 4242, processId: "jobs-main", releaseId: "v1", timeoutMs: 40}}
|
|
256
|
+
])
|
|
257
|
+
|
|
258
|
+
// Shutdown has no budget to hand out and must still wait for the processes it owns, so an
|
|
259
|
+
// unbudgeted stop changes nothing.
|
|
260
|
+
logs.length = 0
|
|
261
|
+
stopOptions.length = 0
|
|
262
|
+
await release.stop()
|
|
263
|
+
|
|
264
|
+
expect(stopOptions).toEqual([
|
|
265
|
+
{id: "worker", options: {}},
|
|
266
|
+
{id: "jobs-main", options: {}},
|
|
267
|
+
{id: "web", options: {}}
|
|
268
|
+
])
|
|
269
|
+
expect(logs).toHaveLength(0)
|
|
270
|
+
})
|
|
271
|
+
|
|
112
272
|
test("restoring a stopped release does not re-adopt a stale connection count", async () => {
|
|
113
273
|
const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}})
|
|
114
274
|
|
|
@@ -118,6 +278,7 @@ test("restoring a stopped release does not re-adopt a stale connection count", a
|
|
|
118
278
|
cwd: "/tmp/rel",
|
|
119
279
|
exitCode: 143,
|
|
120
280
|
exitSignal: "SIGTERM",
|
|
281
|
+
heldProcessGroup: undefined,
|
|
121
282
|
id: "web",
|
|
122
283
|
lastMemoryRestartAt: undefined,
|
|
123
284
|
lastStartReason: "deploy",
|
|
@@ -125,6 +286,7 @@ test("restoring a stopped release does not re-adopt a stale connection count", a
|
|
|
125
286
|
memoryRestarts: 0,
|
|
126
287
|
pid: undefined,
|
|
127
288
|
restarts: 0,
|
|
289
|
+
retirementTimedOut: false,
|
|
128
290
|
rssBytes: undefined,
|
|
129
291
|
startedAt: "2026-10-08T06:22:23.500Z",
|
|
130
292
|
state: "stopped",
|
|
@@ -135,6 +297,7 @@ test("restoring a stopped release does not re-adopt a stale connection count", a
|
|
|
135
297
|
activatedAt: "2026-10-08T06:22:23.583Z",
|
|
136
298
|
connectionCount: 14,
|
|
137
299
|
connections: {http: 14, websocket: 0},
|
|
300
|
+
drain: undefined,
|
|
138
301
|
drainStartedAt: "2026-10-08T07:58:00.464Z",
|
|
139
302
|
ports: {web: 4200},
|
|
140
303
|
processes: [processStatus],
|
|
@@ -150,4 +313,286 @@ test("restoring a stopped release does not re-adopt a stale connection count", a
|
|
|
150
313
|
expect(release.status().connectionCount).toBe(0)
|
|
151
314
|
expect(release.localConnections()).toEqual({http: 0, websocket: 0})
|
|
152
315
|
})
|
|
316
|
+
|
|
317
|
+
test("status reports the drain classification of a draining generation and none of an active one", () => {
|
|
318
|
+
const procPath = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-drain-status-"))
|
|
319
|
+
|
|
320
|
+
try {
|
|
321
|
+
writeProcMember(procPath, 101, 101, "S", 300, 200)
|
|
322
|
+
writeProcMember(procPath, 102, 101, "S", 10, 5)
|
|
323
|
+
|
|
324
|
+
const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}}, () => true, procPath)
|
|
325
|
+
|
|
326
|
+
release.state = "draining"
|
|
327
|
+
release.processes.set("web", drainedProcess(101))
|
|
328
|
+
|
|
329
|
+
const first = release.status().drain
|
|
330
|
+
|
|
331
|
+
expect(first?.status).toBe("unproven")
|
|
332
|
+
expect(first?.sampleWindowMs).toBe(0)
|
|
333
|
+
expect(first?.reason).toBe("first liveness observation; a 20000ms window is required before CPU flatness can be claimed")
|
|
334
|
+
|
|
335
|
+
release.state = "active"
|
|
336
|
+
|
|
337
|
+
expect(release.status().drain).toBe(undefined)
|
|
338
|
+
} finally {
|
|
339
|
+
fs.rmSync(procPath, {force: true, recursive: true})
|
|
340
|
+
}
|
|
341
|
+
})
|
|
342
|
+
|
|
343
|
+
test("a draining generation that still owns a connection past its drain budget is named timedOut, not left claiming progress", () => {
|
|
344
|
+
const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}}, () => true)
|
|
345
|
+
|
|
346
|
+
release.state = "draining"
|
|
347
|
+
release.connectionCount = 1
|
|
348
|
+
// The connection wait was given up at proxy.drainTimeoutMs (60000ms default) 30 seconds ago.
|
|
349
|
+
release.drainStartedAt = new Date(Date.now() - 90000).toISOString()
|
|
350
|
+
|
|
351
|
+
const pastBudget = release.status().drain
|
|
352
|
+
|
|
353
|
+
expect(pastBudget?.status).toBe("timedOut")
|
|
354
|
+
expect(pastBudget?.reason).toMatch(/^drain budget 60000ms exceeded by \d+ms with 1 live connection still owned by the generation$/)
|
|
355
|
+
|
|
356
|
+
release.drainStartedAt = new Date().toISOString()
|
|
357
|
+
|
|
358
|
+
const insideBudget = release.status().drain
|
|
359
|
+
|
|
360
|
+
expect(insideBudget?.status).toBe("progressing")
|
|
361
|
+
expect(insideBudget?.reason).toBe("1 live connection still owned by the generation")
|
|
362
|
+
})
|
|
363
|
+
|
|
364
|
+
test("a drain observation is reused until a full sample window has passed", () => {
|
|
365
|
+
const procPath = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-drain-window-"))
|
|
366
|
+
|
|
367
|
+
try {
|
|
368
|
+
writeProcMember(procPath, 101, 101, "S", 300, 200)
|
|
369
|
+
writeProcMember(procPath, 102, 101, "S", 10, 5)
|
|
370
|
+
|
|
371
|
+
const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}}, () => true, procPath)
|
|
372
|
+
|
|
373
|
+
release.state = "draining"
|
|
374
|
+
release.processes.set("web", drainedProcess(101))
|
|
375
|
+
|
|
376
|
+
expect(release.status().drain?.status).toBe("unproven")
|
|
377
|
+
|
|
378
|
+
// Backdate the newest observation so the next read is a full window behind it.
|
|
379
|
+
release.drainNewestSample = {
|
|
380
|
+
atMs: Date.now() - 25000,
|
|
381
|
+
processes: [{cpuMs: 5150, heldProcessGroupPgid: undefined, id: "web", liveChildCount: 1, pid: 101, retirementTimedOut: true}]
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
const second = release.status().drain
|
|
385
|
+
|
|
386
|
+
expect(second?.status).toBe("wedged")
|
|
387
|
+
expect(second?.sampleWindowMs).toBe(25000)
|
|
388
|
+
expect(second?.cpuDeltaMs).toBe(0)
|
|
389
|
+
|
|
390
|
+
// The observation taken above is now the newest one. Changing the fixture inside the window
|
|
391
|
+
// must not produce a second observation: the classification still reports the reused sample,
|
|
392
|
+
// which is what keeps a procfs scan off the status path.
|
|
393
|
+
writeProcMember(procPath, 101, 101, "S", 900, 600)
|
|
394
|
+
|
|
395
|
+
const third = release.status().drain
|
|
396
|
+
|
|
397
|
+
expect(third?.status).toBe("wedged")
|
|
398
|
+
expect(third?.sampleWindowMs).toBe(25000)
|
|
399
|
+
expect(third?.cpuDeltaMs).toBe(0)
|
|
400
|
+
} finally {
|
|
401
|
+
fs.rmSync(procPath, {force: true, recursive: true})
|
|
402
|
+
}
|
|
403
|
+
})
|
|
404
|
+
|
|
405
|
+
test("a generation still holding a process group after its tracked process exited is observed through that group", () => {
|
|
406
|
+
const procPath = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-drain-held-"))
|
|
407
|
+
|
|
408
|
+
try {
|
|
409
|
+
// The group leader exited, so only the member that outlived it is left in the group.
|
|
410
|
+
writeProcMember(procPath, 102, 101, "S", 300, 200)
|
|
411
|
+
|
|
412
|
+
const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}}, () => true, procPath)
|
|
413
|
+
|
|
414
|
+
release.state = "draining"
|
|
415
|
+
release.processes.set("web", drainedProcess(undefined, {liveMemberCount: 1, pgid: 101}))
|
|
416
|
+
|
|
417
|
+
const drain = release.status().drain
|
|
418
|
+
|
|
419
|
+
expect(drain?.status).toBe("wedged")
|
|
420
|
+
expect(drain?.reason).toBe("1 live process past its retirement budget with no runner child remaining; 1 tracked process already exited and left web pgid 101 alive")
|
|
421
|
+
} finally {
|
|
422
|
+
fs.rmSync(procPath, {force: true, recursive: true})
|
|
423
|
+
}
|
|
424
|
+
})
|
|
425
|
+
|
|
426
|
+
/**
|
|
427
|
+
* @param {number} pid - Live pid the supervisor can still signal.
|
|
428
|
+
* @param {string[]} escalations - Collects the reason of every escalation delivered to this process.
|
|
429
|
+
* @returns {import("../src/managed-process.js").default} A process whose retirement wait already expired.
|
|
430
|
+
*/
|
|
431
|
+
function wedgedProcess(pid, escalations) {
|
|
432
|
+
// eslint-disable-next-line local/no-unknown-jsdoc-type
|
|
433
|
+
return /** @type {import("../src/managed-process.js").default} */ (/** @type {unknown} */ ({
|
|
434
|
+
escalateRetirement: async (/** @type {{reason: string}} */ options) => {
|
|
435
|
+
escalations.push(options.reason)
|
|
436
|
+
|
|
437
|
+
return true
|
|
438
|
+
},
|
|
439
|
+
quiesceStrict: async () => {},
|
|
440
|
+
retirementTimedOut: true,
|
|
441
|
+
status: () => ({pid, retirementTimedOut: true, state: "quiesced"}),
|
|
442
|
+
stop: async () => {}
|
|
443
|
+
}))
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
test("the supervisor escalates a wedged generation whose residue holds no runner child", async () => {
|
|
447
|
+
const procPath = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-escalate-wedged-"))
|
|
448
|
+
|
|
449
|
+
try {
|
|
450
|
+
// Group leader only, 5000ms of consumed CPU: nothing left in the group but the wedged worker.
|
|
451
|
+
writeProcMember(procPath, 101, 101, "S", 300, 200)
|
|
452
|
+
|
|
453
|
+
/** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
|
|
454
|
+
const logs = []
|
|
455
|
+
/** @type {string[]} */
|
|
456
|
+
const escalations = []
|
|
457
|
+
const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}}, () => true, procPath)
|
|
458
|
+
|
|
459
|
+
release.logger = (message, data) => logs.push({data, message})
|
|
460
|
+
release.state = "stopped"
|
|
461
|
+
release.processes.set("worker", wedgedProcess(101, escalations))
|
|
462
|
+
// The generation already recorded `release drained`; this is the observation its retirement
|
|
463
|
+
// left behind, a full window behind the read that now acts on it.
|
|
464
|
+
release.drainNewestSample = {
|
|
465
|
+
atMs: Date.now() - 25000,
|
|
466
|
+
processes: [{cpuMs: 5000, heldProcessGroupPgid: undefined, id: "worker", liveChildCount: 0, pid: 101, retirementTimedOut: true}]
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
const escalated = await release.escalateWedgedGeneration({reason: "deploy of v2"})
|
|
470
|
+
|
|
471
|
+
expect(escalated).toEqual(["worker"])
|
|
472
|
+
expect(escalations).toHaveLength(1)
|
|
473
|
+
expect(escalations[0]).toContain("deploy of v2")
|
|
474
|
+
expect(escalations[0]).toContain("no runner child remaining")
|
|
475
|
+
expect(logs).toMatchObject([{data: {escalation: "SIGKILL", processIds: ["worker"], releaseId: "v1", trigger: "deploy of v2"}, message: "release wedged generation escalated"}])
|
|
476
|
+
} finally {
|
|
477
|
+
fs.rmSync(procPath, {force: true, recursive: true})
|
|
478
|
+
}
|
|
479
|
+
})
|
|
480
|
+
|
|
481
|
+
test("a wedged generation that still holds a live runner child is reported, never killed", async () => {
|
|
482
|
+
const procPath = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-escalate-refused-"))
|
|
483
|
+
|
|
484
|
+
try {
|
|
485
|
+
writeProcMember(procPath, 101, 101, "S", 300, 200)
|
|
486
|
+
// A live runner child: the wedge is real, and killing it is still the STOE-1077 failure.
|
|
487
|
+
writeProcMember(procPath, 102, 101, "S", 0, 0)
|
|
488
|
+
|
|
489
|
+
/** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
|
|
490
|
+
const logs = []
|
|
491
|
+
/** @type {string[]} */
|
|
492
|
+
const escalations = []
|
|
493
|
+
const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}}, () => true, procPath)
|
|
494
|
+
|
|
495
|
+
release.logger = (message, data) => logs.push({data, message})
|
|
496
|
+
release.state = "stopped"
|
|
497
|
+
release.processes.set("worker", wedgedProcess(101, escalations))
|
|
498
|
+
release.drainNewestSample = {
|
|
499
|
+
atMs: Date.now() - 25000,
|
|
500
|
+
processes: [{cpuMs: 5000, heldProcessGroupPgid: undefined, id: "worker", liveChildCount: 1, pid: 101, retirementTimedOut: true}]
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
const escalated = await release.escalateWedgedGeneration({reason: "deploy of v2"})
|
|
504
|
+
|
|
505
|
+
expect(escalated).toEqual([])
|
|
506
|
+
expect(escalations).toEqual([])
|
|
507
|
+
expect(logs).toMatchObject([{data: {escalation: "refused", pids: [101], processIds: ["worker"], releaseId: "v1"}, message: "release wedged generation left alive"}])
|
|
508
|
+
expect(logs[0].data?.reason).toBe("no escalation: 1 runner child is still live in the generation; SIGKILL would abort in-flight work")
|
|
509
|
+
} finally {
|
|
510
|
+
fs.rmSync(procPath, {force: true, recursive: true})
|
|
511
|
+
}
|
|
512
|
+
})
|
|
513
|
+
|
|
514
|
+
test("the retirement transition brackets its own wait and reports a residue it cannot yet prove wedged", async () => {
|
|
515
|
+
const procPath = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-escalate-bracket-"))
|
|
516
|
+
|
|
517
|
+
try {
|
|
518
|
+
writeProcMember(procPath, 4242, 4242, "S", 300, 200)
|
|
519
|
+
|
|
520
|
+
/** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
|
|
521
|
+
const logs = []
|
|
522
|
+
/** @type {string[]} */
|
|
523
|
+
const escalations = []
|
|
524
|
+
const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}}, () => true, procPath)
|
|
525
|
+
|
|
526
|
+
release.logger = (message, data) => logs.push({data, message})
|
|
527
|
+
release.state = "draining"
|
|
528
|
+
release.nonBlockingDrainIds.add("worker")
|
|
529
|
+
release.handoffServiceIds.add("jobs-main")
|
|
530
|
+
release.processes.set("worker", wedgedProcess(4242, escalations))
|
|
531
|
+
// eslint-disable-next-line local/no-unknown-jsdoc-type
|
|
532
|
+
release.processes.set("jobs-main", /** @type {import("../src/managed-process.js").default} */ (/** @type {unknown} */ ({
|
|
533
|
+
status: () => ({pid: undefined, retirementTimedOut: false, state: "stopped"}),
|
|
534
|
+
stop: async () => {}
|
|
535
|
+
})))
|
|
536
|
+
|
|
537
|
+
await release.drainAndStop(0, {
|
|
538
|
+
...release.config,
|
|
539
|
+
proxy: {...release.config.proxy, nonBlockingDrainTimeoutMs: 20},
|
|
540
|
+
releaseRetention: {...release.config.releaseRetention, escalateWedgedGenerations: true}
|
|
541
|
+
})
|
|
542
|
+
|
|
543
|
+
// The transition still completes: the escalation is its last act, never a wait of its own.
|
|
544
|
+
expect(release.state).toBe("stopped")
|
|
545
|
+
expect(escalations).toEqual([])
|
|
546
|
+
expect(release.drainPriorSample).toBeDefined()
|
|
547
|
+
expect(release.drainNewestSample).toBeDefined()
|
|
548
|
+
expect(logs).toMatchObject([
|
|
549
|
+
{data: {pid: 4242, processId: "worker"}, message: "release non-blocking process still draining"},
|
|
550
|
+
{data: {escalation: "refused", pids: [4242], releaseId: "v1", trigger: "release retirement"}, message: "release wedged generation left alive"}
|
|
551
|
+
])
|
|
552
|
+
// A bracket shorter than the flatness window cannot justify a kill; the retained pair is
|
|
553
|
+
// what lets the next deploy prove the residue without sampling on a timer.
|
|
554
|
+
expect(logs[1].data?.reason).toContain("is below the 20000ms minimum")
|
|
555
|
+
} finally {
|
|
556
|
+
fs.rmSync(procPath, {force: true, recursive: true})
|
|
557
|
+
}
|
|
558
|
+
})
|
|
559
|
+
|
|
560
|
+
test("retirement stores the refreshed definition identity it applied to the live generation", async () => {
|
|
561
|
+
const config = normalizeConfig({
|
|
562
|
+
application: "demo",
|
|
563
|
+
control: {path: "/tmp/rollbridge-release-group.sock"},
|
|
564
|
+
processes: [
|
|
565
|
+
{command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}},
|
|
566
|
+
{command: "run jobs", deployStrategy: "handoff", id: "jobs", policy: "service", port: {from: 0, to: 1}}
|
|
567
|
+
],
|
|
568
|
+
proxy: {host: "127.0.0.1", port: 0}
|
|
569
|
+
})
|
|
570
|
+
const release = new ReleaseGroup({config, logger: () => {}, procPath: "/proc", releaseId: "v1", releasePath: "/tmp/rel", revision: "v1", shouldStart: () => true})
|
|
571
|
+
const generationConfig = release.config
|
|
572
|
+
/** @type {Array<import("../src/managed-process.js").ManagedProcessDefinition>} */
|
|
573
|
+
const refreshed = []
|
|
574
|
+
|
|
575
|
+
// eslint-disable-next-line local/no-unknown-jsdoc-type
|
|
576
|
+
release.processes.set("jobs", /** @type {import("../src/managed-process.js").default} */ (/** @type {unknown} */ ({
|
|
577
|
+
id: "jobs:v1",
|
|
578
|
+
quiesceStrict: async () => {},
|
|
579
|
+
status: () => ({pid: 4242, state: "running"}),
|
|
580
|
+
updateDefinition: (/** @type {import("../src/managed-process.js").ManagedProcessDefinition} */ definition) => { refreshed.push(definition) }
|
|
581
|
+
})))
|
|
582
|
+
|
|
583
|
+
const successor = {...release.config, processes: [release.config.processes[0], {...release.config.processes[1], env: {RELEASE_CONFIG_AUTHORITY: "v2"}}]}
|
|
584
|
+
|
|
585
|
+
await release.beginRetirement(successor)
|
|
586
|
+
|
|
587
|
+
// The live process is quiesced with the successor's definition, so the guardian records the
|
|
588
|
+
// successor's provenance for it. The generation's stored config must move with that definition:
|
|
589
|
+
// the durable owner state publishes it as the config recovery rebuilds each live process from, and
|
|
590
|
+
// a retained pre-refresh config makes the next recovery re-register the live process with the
|
|
591
|
+
// stale provenance and abort (STOE-2040).
|
|
592
|
+
expect(refreshed.map((definition) => definition.env?.RELEASE_CONFIG_AUTHORITY)).toEqual(["v2"])
|
|
593
|
+
expect(release.handoffServiceIds.has("jobs")).toBe(true)
|
|
594
|
+
expect(release.config).toBe(successor)
|
|
595
|
+
expect(release.config).not.toBe(generationConfig)
|
|
596
|
+
expect(release.state).toBe("draining")
|
|
597
|
+
})
|
|
153
598
|
})
|