rollbridge 0.1.65 → 0.1.67
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +27 -0
- package/changelog.d/20261115-wedged-generation-escalation.md +44 -0
- package/changelog.d/20261116-bounded-explicit-stop-retirement.md +30 -0
- package/docs/cli.md +16 -1
- package/docs/config.md +47 -1
- package/docs/troubleshooting.md +24 -0
- package/docs/workers.md +10 -0
- package/package.json +1 -1
- package/src/config.js +2 -1
- package/src/daemon.js +124 -24
- package/src/drain-policy.js +79 -0
- package/src/managed-process.js +71 -6
- package/src/release-group.js +146 -22
- package/test/config-validation.test.js +5 -4
- package/test/drain-policy.test.js +102 -1
- package/test/managed-process.test.js +205 -1
- package/test/release-group.test.js +196 -0
- package/test/rollbridge.test.js +173 -0
|
@@ -309,10 +309,12 @@ test("runs quiet and drain lifecycle hooks before stopping", async () => {
|
|
|
309
309
|
})
|
|
310
310
|
|
|
311
311
|
test("retirement timeout leaves an indefinite worker alive without SIGKILL", async () => {
|
|
312
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-retirement-wedge-"))
|
|
313
|
+
const readyPath = path.join(dir, "ready")
|
|
312
314
|
/** @type {string[]} */
|
|
313
315
|
const signals = []
|
|
314
316
|
const managed = new ManagedProcess({
|
|
315
|
-
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(
|
|
317
|
+
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`process.on('SIGTERM', () => {}); require("node:fs").writeFileSync(${JSON.stringify(readyPath)}, "ready"); setInterval(() => {}, 1000)`)}`,
|
|
316
318
|
cwd: undefined,
|
|
317
319
|
env: {},
|
|
318
320
|
id: "worker",
|
|
@@ -332,8 +334,210 @@ test("retirement timeout leaves an indefinite worker alive without SIGKILL", asy
|
|
|
332
334
|
|
|
333
335
|
try {
|
|
334
336
|
await managed.start()
|
|
337
|
+
// The fixture installs its SIGTERM disposition before it announces readiness. Signalling a
|
|
338
|
+
// child that has not finished booting reaches the default disposition instead, which exits
|
|
339
|
+
// the group and leaves no wedge to observe.
|
|
340
|
+
await waitFor(() => fs.existsSync(readyPath))
|
|
341
|
+
await managed.stop({retirementTimeoutMs: 25})
|
|
342
|
+
|
|
343
|
+
expect(managed.retirementTimedOut).toBe(true)
|
|
344
|
+
expect(signals).toEqual(["SIGTERM"])
|
|
345
|
+
expect(managed.status().pid).toBeDefined()
|
|
346
|
+
} finally {
|
|
347
|
+
const pid = managed.status().pid
|
|
348
|
+
if (pid) process.kill(-pid, "SIGKILL")
|
|
349
|
+
if (managed.exitPromise) await managed.exitPromise
|
|
350
|
+
fs.rmSync(dir, {force: true, recursive: true})
|
|
351
|
+
}
|
|
352
|
+
})
|
|
353
|
+
|
|
354
|
+
test("escalateRetirement terminates the wedged worker the bounded retirement left alive", async () => {
|
|
355
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-escalation-wedge-"))
|
|
356
|
+
const readyPath = path.join(dir, "ready")
|
|
357
|
+
/** @type {string[]} */
|
|
358
|
+
const signals = []
|
|
359
|
+
/** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
|
|
360
|
+
const logs = []
|
|
361
|
+
const managed = new ManagedProcess({
|
|
362
|
+
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`process.on('SIGTERM', () => {}); require("node:fs").writeFileSync(${JSON.stringify(readyPath)}, "ready"); setInterval(() => {}, 1000)`)}`,
|
|
363
|
+
cwd: undefined,
|
|
364
|
+
env: {},
|
|
365
|
+
id: "worker",
|
|
366
|
+
lifecycle: {drainTimeoutMs: 0},
|
|
367
|
+
logger: (message, data) => logs.push({data, message}),
|
|
368
|
+
outputLines: 50,
|
|
369
|
+
restartDelayMs: 10,
|
|
370
|
+
shouldRestart: () => false,
|
|
371
|
+
stopSignal: "SIGTERM",
|
|
372
|
+
stopTimeoutMs: "indefinite"
|
|
373
|
+
})
|
|
374
|
+
const signalProcessGroup = managed.signalProcessGroup.bind(managed)
|
|
375
|
+
|
|
376
|
+
managed.signalProcessGroup = async (signal, pgid, deadline) => {
|
|
377
|
+
signals.push(signal)
|
|
378
|
+
await signalProcessGroup(signal, pgid, deadline)
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
try {
|
|
382
|
+
await managed.start()
|
|
383
|
+
await waitFor(() => fs.existsSync(readyPath))
|
|
335
384
|
await managed.stop({retirementTimeoutMs: 25})
|
|
385
|
+
const wedgedPid = managed.status().pid
|
|
386
|
+
|
|
387
|
+
expect(managed.retirementTimedOut).toBe(true)
|
|
388
|
+
expect(signals).toEqual(["SIGTERM"])
|
|
389
|
+
expect(wedgedPid).toBeDefined()
|
|
390
|
+
|
|
391
|
+
expect(await managed.escalateRetirement({reason: "spec: proven wedge"})).toBe(true)
|
|
392
|
+
expect(signals).toEqual(["SIGTERM", "SIGKILL"])
|
|
393
|
+
expect(logs.some((entry) => entry.message === "process retirement escalated to SIGKILL" && entry.data?.reason === "spec: proven wedge")).toBe(true)
|
|
394
|
+
expect(await managed.waitForProcessGroupExit(/** @type {number} */ (wedgedPid), Date.now() + 5000)).toBe(true)
|
|
395
|
+
if (managed.exitPromise) await managed.exitPromise
|
|
396
|
+
expect(managed.status().pid).toBeUndefined()
|
|
397
|
+
expect(managed.status().state).toBe("stopped")
|
|
398
|
+
} finally {
|
|
399
|
+
const pid = managed.status().pid
|
|
400
|
+
if (pid) process.kill(-pid, "SIGKILL")
|
|
401
|
+
if (managed.exitPromise) await managed.exitPromise
|
|
402
|
+
fs.rmSync(dir, {force: true, recursive: true})
|
|
403
|
+
}
|
|
404
|
+
})
|
|
405
|
+
|
|
406
|
+
test("escalateRetirement refuses a process that has not expired its retirement budget", async () => {
|
|
407
|
+
/** @type {string[]} */
|
|
408
|
+
const signals = []
|
|
409
|
+
/** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
|
|
410
|
+
const logs = []
|
|
411
|
+
const managed = new ManagedProcess({
|
|
412
|
+
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify("setInterval(() => {}, 1000)")}`,
|
|
413
|
+
cwd: undefined,
|
|
414
|
+
env: {},
|
|
415
|
+
id: "worker",
|
|
416
|
+
lifecycle: {drainTimeoutMs: 0},
|
|
417
|
+
logger: (message, data) => logs.push({data, message}),
|
|
418
|
+
outputLines: 50,
|
|
419
|
+
restartDelayMs: 10,
|
|
420
|
+
shouldRestart: () => false,
|
|
421
|
+
stopSignal: "SIGTERM",
|
|
422
|
+
stopTimeoutMs: "indefinite"
|
|
423
|
+
})
|
|
424
|
+
const signalProcessGroup = managed.signalProcessGroup.bind(managed)
|
|
336
425
|
|
|
426
|
+
managed.signalProcessGroup = async (signal, pgid, deadline) => {
|
|
427
|
+
signals.push(signal)
|
|
428
|
+
await signalProcessGroup(signal, pgid, deadline)
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
try {
|
|
432
|
+
await managed.start()
|
|
433
|
+
await managed.quiesce()
|
|
434
|
+
|
|
435
|
+
expect(managed.retirementTimedOut).toBe(false)
|
|
436
|
+
expect(await managed.escalateRetirement({reason: "spec"})).toBe(false)
|
|
437
|
+
expect(signals).toEqual([])
|
|
438
|
+
expect(logs.some((entry) => entry.message === "retirement escalation refused; process is not past its retirement budget")).toBe(true)
|
|
439
|
+
expect(managed.status().pid).toBeDefined()
|
|
440
|
+
} finally {
|
|
441
|
+
const pid = managed.status().pid
|
|
442
|
+
if (pid) process.kill(-pid, "SIGKILL")
|
|
443
|
+
if (managed.exitPromise) await managed.exitPromise
|
|
444
|
+
}
|
|
445
|
+
})
|
|
446
|
+
|
|
447
|
+
test("escalateRetirement is inert for a process that drained normally", async () => {
|
|
448
|
+
/** @type {string[]} */
|
|
449
|
+
const signals = []
|
|
450
|
+
const managed = new ManagedProcess({
|
|
451
|
+
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify("setInterval(() => {}, 1000)")}`,
|
|
452
|
+
cwd: undefined,
|
|
453
|
+
env: {},
|
|
454
|
+
id: "worker",
|
|
455
|
+
lifecycle: {drainTimeoutMs: 0},
|
|
456
|
+
logger: () => {},
|
|
457
|
+
outputLines: 50,
|
|
458
|
+
restartDelayMs: 10,
|
|
459
|
+
shouldRestart: () => false,
|
|
460
|
+
stopSignal: "SIGTERM",
|
|
461
|
+
stopTimeoutMs: 2000
|
|
462
|
+
})
|
|
463
|
+
const signalProcessGroup = managed.signalProcessGroup.bind(managed)
|
|
464
|
+
|
|
465
|
+
managed.signalProcessGroup = async (signal, pgid, deadline) => {
|
|
466
|
+
signals.push(signal)
|
|
467
|
+
await signalProcessGroup(signal, pgid, deadline)
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
try {
|
|
471
|
+
await managed.start()
|
|
472
|
+
await managed.stop()
|
|
473
|
+
|
|
474
|
+
expect(managed.status().state).toBe("stopped")
|
|
475
|
+
expect(await managed.escalateRetirement({reason: "spec"})).toBe(false)
|
|
476
|
+
expect(signals).toEqual(["SIGTERM"])
|
|
477
|
+
} finally {
|
|
478
|
+
const pid = managed.status().pid
|
|
479
|
+
if (pid) process.kill(-pid, "SIGKILL")
|
|
480
|
+
if (managed.exitPromise) await managed.exitPromise
|
|
481
|
+
}
|
|
482
|
+
})
|
|
483
|
+
|
|
484
|
+
test("a retirement budget caps a finite graceful window instead of being outvoted by it", async () => {
|
|
485
|
+
/** @type {string[]} */
|
|
486
|
+
const signals = []
|
|
487
|
+
/** @type {Array<{msFromStopStart: number, phase: string}>} */
|
|
488
|
+
const deadlines = []
|
|
489
|
+
const managed = new ManagedProcess({
|
|
490
|
+
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify("process.on('SIGTERM', () => {}); setInterval(() => {}, 1000)")}`,
|
|
491
|
+
cwd: undefined,
|
|
492
|
+
env: {},
|
|
493
|
+
id: "worker",
|
|
494
|
+
lifecycle: {drainTimeoutMs: 0},
|
|
495
|
+
logger: () => {},
|
|
496
|
+
outputLines: 50,
|
|
497
|
+
restartDelayMs: 10,
|
|
498
|
+
shouldRestart: () => false,
|
|
499
|
+
stopSignal: "SIGTERM",
|
|
500
|
+
stopTimeoutMs: 2000
|
|
501
|
+
})
|
|
502
|
+
let stopStartedAtMs = 0
|
|
503
|
+
|
|
504
|
+
/**
|
|
505
|
+
* @param {string} phase - The step the deadline was handed to.
|
|
506
|
+
* @param {number | undefined} deadline - Absolute deadline the step received.
|
|
507
|
+
* @returns {void} Records the deadline as an offset from the start of the stop.
|
|
508
|
+
*/
|
|
509
|
+
const recordDeadline = (phase, deadline) => {
|
|
510
|
+
if (deadline !== undefined) deadlines.push({msFromStopStart: deadline - stopStartedAtMs, phase})
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
const signalProcessGroup = managed.signalProcessGroup.bind(managed)
|
|
514
|
+
managed.signalProcessGroup = async (signal, pgid, deadline) => {
|
|
515
|
+
signals.push(signal)
|
|
516
|
+
recordDeadline("signal", deadline)
|
|
517
|
+
await signalProcessGroup(signal, pgid, deadline)
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
const waitForProcessGroupExit = managed.waitForProcessGroupExit.bind(managed)
|
|
521
|
+
managed.waitForProcessGroupExit = async (pgid, deadline) => {
|
|
522
|
+
recordDeadline("wait", deadline)
|
|
523
|
+
return await waitForProcessGroupExit(pgid, deadline)
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
try {
|
|
527
|
+
await managed.start()
|
|
528
|
+
stopStartedAtMs = Date.now()
|
|
529
|
+
|
|
530
|
+
await managed.stop({retirementTimeoutMs: 25})
|
|
531
|
+
|
|
532
|
+
// The 2 000 ms graceful window must not outvote the 25 ms retirement budget. Read the other
|
|
533
|
+
// way round the wait runs the whole window before it escalates to SIGKILL, which is the shape
|
|
534
|
+
// that held the TensorBuzz control plane for 4,5 h after the worker moved from "indefinite" to
|
|
535
|
+
// 4,5 h and killed the in-flight builds the consumer said not to interrupt.
|
|
536
|
+
expect(deadlines.map((entry) => entry.phase)).toEqual(["signal", "wait"])
|
|
537
|
+
for (const entry of deadlines) {
|
|
538
|
+
expect(entry.msFromStopStart).toBeGreaterThanOrEqual(25)
|
|
539
|
+
expect(entry.msFromStopStart).toBeLessThan(1000)
|
|
540
|
+
}
|
|
337
541
|
expect(managed.retirementTimedOut).toBe(true)
|
|
338
542
|
expect(signals).toEqual(["SIGTERM"])
|
|
339
543
|
expect(managed.status().pid).toBeDefined()
|
|
@@ -171,6 +171,68 @@ test("a wedged non-blocking process cannot block release retirement", async () =
|
|
|
171
171
|
expect(logs).toMatchObject([{message: "release non-blocking process still draining", data: {pid: 4242, processId: "worker"}}])
|
|
172
172
|
})
|
|
173
173
|
|
|
174
|
+
test("an explicit stop bounds the processes it must not kill and reports what it left alive", async () => {
|
|
175
|
+
/** @type {Array<{message: string, data?: import("../src/json.js").JsonValue}>} */
|
|
176
|
+
const logs = []
|
|
177
|
+
/** @type {Array<{id: string, options: {retirementTimeoutMs?: number, timeoutMs?: number}}>} */
|
|
178
|
+
const stopOptions = []
|
|
179
|
+
const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}})
|
|
180
|
+
|
|
181
|
+
release.logger = (message, data) => logs.push({message, data})
|
|
182
|
+
release.nonBlockingDrainIds.add("worker")
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* @param {string} id - Process id.
|
|
186
|
+
* @param {boolean} unboundedWindow - Whether its graceful window can ever end.
|
|
187
|
+
* @returns {void} A process recorded in the generation.
|
|
188
|
+
*/
|
|
189
|
+
const addProcess = (id, unboundedWindow) => {
|
|
190
|
+
const instance = {
|
|
191
|
+
hasUnboundedGracefulWindow: () => unboundedWindow,
|
|
192
|
+
retirementTimedOut: false,
|
|
193
|
+
status: () => ({pid: 4242, state: "stopped"}),
|
|
194
|
+
stop: async (/** @type {{retirementTimeoutMs?: number, timeoutMs?: number}} */ options) => {
|
|
195
|
+
stopOptions.push({id, options})
|
|
196
|
+
// Only the retirement branch sets this, so a process that was never budgeted never claims it.
|
|
197
|
+
if (options.retirementTimeoutMs !== undefined) instance.retirementTimedOut = true
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
// eslint-disable-next-line local/no-unknown-jsdoc-type
|
|
202
|
+
release.processes.set(id, /** @type {import("../src/managed-process.js").default} */ (/** @type {unknown} */ (instance)))
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
addProcess("worker", false)
|
|
206
|
+
addProcess("jobs-main", true)
|
|
207
|
+
addProcess("web", false)
|
|
208
|
+
|
|
209
|
+
await release.stop({retirementTimeoutMs: 40})
|
|
210
|
+
|
|
211
|
+
expect(stopOptions).toEqual([
|
|
212
|
+
{id: "worker", options: {retirementTimeoutMs: 40}},
|
|
213
|
+
{id: "jobs-main", options: {retirementTimeoutMs: 40}},
|
|
214
|
+
{id: "web", options: {}}
|
|
215
|
+
])
|
|
216
|
+
expect(release.state).toBe("stopped")
|
|
217
|
+
expect(logs).toMatchObject([
|
|
218
|
+
{message: "release stop gave up waiting for a retiring process", data: {pid: 4242, processId: "worker", releaseId: "v1", timeoutMs: 40}},
|
|
219
|
+
{message: "release stop gave up waiting for a retiring process", data: {pid: 4242, processId: "jobs-main", releaseId: "v1", timeoutMs: 40}}
|
|
220
|
+
])
|
|
221
|
+
|
|
222
|
+
// Shutdown has no budget to hand out and must still wait for the processes it owns, so an
|
|
223
|
+
// unbudgeted stop changes nothing.
|
|
224
|
+
logs.length = 0
|
|
225
|
+
stopOptions.length = 0
|
|
226
|
+
await release.stop()
|
|
227
|
+
|
|
228
|
+
expect(stopOptions).toEqual([
|
|
229
|
+
{id: "worker", options: {}},
|
|
230
|
+
{id: "jobs-main", options: {}},
|
|
231
|
+
{id: "web", options: {}}
|
|
232
|
+
])
|
|
233
|
+
expect(logs).toHaveLength(0)
|
|
234
|
+
})
|
|
235
|
+
|
|
174
236
|
test("restoring a stopped release does not re-adopt a stale connection count", async () => {
|
|
175
237
|
const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}})
|
|
176
238
|
|
|
@@ -303,4 +365,138 @@ test("a generation still holding a process group after its tracked process exite
|
|
|
303
365
|
fs.rmSync(procPath, {force: true, recursive: true})
|
|
304
366
|
}
|
|
305
367
|
})
|
|
368
|
+
|
|
369
|
+
/**
|
|
370
|
+
* @param {number} pid - Live pid the supervisor can still signal.
|
|
371
|
+
* @param {string[]} escalations - Collects the reason of every escalation delivered to this process.
|
|
372
|
+
* @returns {import("../src/managed-process.js").default} A process whose retirement wait already expired.
|
|
373
|
+
*/
|
|
374
|
+
function wedgedProcess(pid, escalations) {
|
|
375
|
+
// eslint-disable-next-line local/no-unknown-jsdoc-type
|
|
376
|
+
return /** @type {import("../src/managed-process.js").default} */ (/** @type {unknown} */ ({
|
|
377
|
+
escalateRetirement: async (/** @type {{reason: string}} */ options) => {
|
|
378
|
+
escalations.push(options.reason)
|
|
379
|
+
|
|
380
|
+
return true
|
|
381
|
+
},
|
|
382
|
+
quiesceStrict: async () => {},
|
|
383
|
+
retirementTimedOut: true,
|
|
384
|
+
status: () => ({pid, retirementTimedOut: true, state: "quiesced"}),
|
|
385
|
+
stop: async () => {}
|
|
386
|
+
}))
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
test("the supervisor escalates a wedged generation whose residue holds no runner child", async () => {
|
|
390
|
+
const procPath = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-escalate-wedged-"))
|
|
391
|
+
|
|
392
|
+
try {
|
|
393
|
+
// Group leader only, 5000ms of consumed CPU: nothing left in the group but the wedged worker.
|
|
394
|
+
writeProcMember(procPath, 101, 101, "S", 300, 200)
|
|
395
|
+
|
|
396
|
+
/** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
|
|
397
|
+
const logs = []
|
|
398
|
+
/** @type {string[]} */
|
|
399
|
+
const escalations = []
|
|
400
|
+
const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}}, () => true, procPath)
|
|
401
|
+
|
|
402
|
+
release.logger = (message, data) => logs.push({data, message})
|
|
403
|
+
release.state = "stopped"
|
|
404
|
+
release.processes.set("worker", wedgedProcess(101, escalations))
|
|
405
|
+
// The generation already recorded `release drained`; this is the observation its retirement
|
|
406
|
+
// left behind, a full window behind the read that now acts on it.
|
|
407
|
+
release.drainNewestSample = {
|
|
408
|
+
atMs: Date.now() - 25000,
|
|
409
|
+
processes: [{cpuMs: 5000, heldProcessGroupPgid: undefined, id: "worker", liveChildCount: 0, pid: 101, retirementTimedOut: true}]
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
const escalated = await release.escalateWedgedGeneration({reason: "deploy of v2"})
|
|
413
|
+
|
|
414
|
+
expect(escalated).toEqual(["worker"])
|
|
415
|
+
expect(escalations).toHaveLength(1)
|
|
416
|
+
expect(escalations[0]).toContain("deploy of v2")
|
|
417
|
+
expect(escalations[0]).toContain("no runner child remaining")
|
|
418
|
+
expect(logs).toMatchObject([{data: {escalation: "SIGKILL", processIds: ["worker"], releaseId: "v1", trigger: "deploy of v2"}, message: "release wedged generation escalated"}])
|
|
419
|
+
} finally {
|
|
420
|
+
fs.rmSync(procPath, {force: true, recursive: true})
|
|
421
|
+
}
|
|
422
|
+
})
|
|
423
|
+
|
|
424
|
+
test("a wedged generation that still holds a live runner child is reported, never killed", async () => {
|
|
425
|
+
const procPath = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-escalate-refused-"))
|
|
426
|
+
|
|
427
|
+
try {
|
|
428
|
+
writeProcMember(procPath, 101, 101, "S", 300, 200)
|
|
429
|
+
// A live runner child: the wedge is real, and killing it is still the STOE-1077 failure.
|
|
430
|
+
writeProcMember(procPath, 102, 101, "S", 0, 0)
|
|
431
|
+
|
|
432
|
+
/** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
|
|
433
|
+
const logs = []
|
|
434
|
+
/** @type {string[]} */
|
|
435
|
+
const escalations = []
|
|
436
|
+
const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}}, () => true, procPath)
|
|
437
|
+
|
|
438
|
+
release.logger = (message, data) => logs.push({data, message})
|
|
439
|
+
release.state = "stopped"
|
|
440
|
+
release.processes.set("worker", wedgedProcess(101, escalations))
|
|
441
|
+
release.drainNewestSample = {
|
|
442
|
+
atMs: Date.now() - 25000,
|
|
443
|
+
processes: [{cpuMs: 5000, heldProcessGroupPgid: undefined, id: "worker", liveChildCount: 1, pid: 101, retirementTimedOut: true}]
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
const escalated = await release.escalateWedgedGeneration({reason: "deploy of v2"})
|
|
447
|
+
|
|
448
|
+
expect(escalated).toEqual([])
|
|
449
|
+
expect(escalations).toEqual([])
|
|
450
|
+
expect(logs).toMatchObject([{data: {escalation: "refused", pids: [101], processIds: ["worker"], releaseId: "v1"}, message: "release wedged generation left alive"}])
|
|
451
|
+
expect(logs[0].data?.reason).toBe("no escalation: 1 runner child is still live in the generation; SIGKILL would abort in-flight work")
|
|
452
|
+
} finally {
|
|
453
|
+
fs.rmSync(procPath, {force: true, recursive: true})
|
|
454
|
+
}
|
|
455
|
+
})
|
|
456
|
+
|
|
457
|
+
test("the retirement transition brackets its own wait and reports a residue it cannot yet prove wedged", async () => {
|
|
458
|
+
const procPath = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-escalate-bracket-"))
|
|
459
|
+
|
|
460
|
+
try {
|
|
461
|
+
writeProcMember(procPath, 4242, 4242, "S", 300, 200)
|
|
462
|
+
|
|
463
|
+
/** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
|
|
464
|
+
const logs = []
|
|
465
|
+
/** @type {string[]} */
|
|
466
|
+
const escalations = []
|
|
467
|
+
const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}}, () => true, procPath)
|
|
468
|
+
|
|
469
|
+
release.logger = (message, data) => logs.push({data, message})
|
|
470
|
+
release.state = "draining"
|
|
471
|
+
release.nonBlockingDrainIds.add("worker")
|
|
472
|
+
release.handoffServiceIds.add("jobs-main")
|
|
473
|
+
release.processes.set("worker", wedgedProcess(4242, escalations))
|
|
474
|
+
// eslint-disable-next-line local/no-unknown-jsdoc-type
|
|
475
|
+
release.processes.set("jobs-main", /** @type {import("../src/managed-process.js").default} */ (/** @type {unknown} */ ({
|
|
476
|
+
status: () => ({pid: undefined, retirementTimedOut: false, state: "stopped"}),
|
|
477
|
+
stop: async () => {}
|
|
478
|
+
})))
|
|
479
|
+
|
|
480
|
+
await release.drainAndStop(0, {
|
|
481
|
+
...release.config,
|
|
482
|
+
proxy: {...release.config.proxy, nonBlockingDrainTimeoutMs: 20},
|
|
483
|
+
releaseRetention: {...release.config.releaseRetention, escalateWedgedGenerations: true}
|
|
484
|
+
})
|
|
485
|
+
|
|
486
|
+
// The transition still completes: the escalation is its last act, never a wait of its own.
|
|
487
|
+
expect(release.state).toBe("stopped")
|
|
488
|
+
expect(escalations).toEqual([])
|
|
489
|
+
expect(release.drainPriorSample).toBeDefined()
|
|
490
|
+
expect(release.drainNewestSample).toBeDefined()
|
|
491
|
+
expect(logs).toMatchObject([
|
|
492
|
+
{data: {pid: 4242, processId: "worker"}, message: "release non-blocking process still draining"},
|
|
493
|
+
{data: {escalation: "refused", pids: [4242], releaseId: "v1", trigger: "release retirement"}, message: "release wedged generation left alive"}
|
|
494
|
+
])
|
|
495
|
+
// A bracket shorter than the flatness window cannot justify a kill; the retained pair is
|
|
496
|
+
// what lets the next deploy prove the residue without sampling on a timer.
|
|
497
|
+
expect(logs[1].data?.reason).toContain("is below the 20000ms minimum")
|
|
498
|
+
} finally {
|
|
499
|
+
fs.rmSync(procPath, {force: true, recursive: true})
|
|
500
|
+
}
|
|
501
|
+
})
|
|
306
502
|
})
|
package/test/rollbridge.test.js
CHANGED
|
@@ -61,6 +61,61 @@ test("a nonBlockingDrain worker stops immediately while its release is still dra
|
|
|
61
61
|
}
|
|
62
62
|
})
|
|
63
63
|
|
|
64
|
+
test("an explicit stop returns within its retirement budget instead of waiting on a worker that is still draining", async () => {
|
|
65
|
+
const fixture = await createFixture({nonBlockingDrainWorker: true, workerStopDelayMs: 30000})
|
|
66
|
+
const config = normalizeConfig({
|
|
67
|
+
...fixture.config,
|
|
68
|
+
processes: fixture.config.processes.map((processConfig) => processConfig.id === "worker"
|
|
69
|
+
? {...processConfig, gracefulStopMs: "indefinite"}
|
|
70
|
+
: processConfig),
|
|
71
|
+
proxy: {...fixture.config.proxy, nonBlockingDrainTimeoutMs: 100}
|
|
72
|
+
})
|
|
73
|
+
/** @type {Array<{message: string, data?: import("../src/json.js").JsonValue}>} */
|
|
74
|
+
const logs = []
|
|
75
|
+
const daemon = new RollbridgeDaemon({config, logger: (message, data) => logs.push({message, data})})
|
|
76
|
+
/** @type {number | undefined} */
|
|
77
|
+
let workerPid
|
|
78
|
+
|
|
79
|
+
try {
|
|
80
|
+
await daemon.start()
|
|
81
|
+
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
82
|
+
|
|
83
|
+
workerPid = statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")?.pid
|
|
84
|
+
|
|
85
|
+
if (typeof workerPid !== "number") throw new Error("Expected the worker to be running before the stop")
|
|
86
|
+
|
|
87
|
+
const startedAtMs = Date.now()
|
|
88
|
+
|
|
89
|
+
await daemon.stopRelease("v1")
|
|
90
|
+
|
|
91
|
+
// Before this the command had no deadline for a worker whose graceful window is "indefinite",
|
|
92
|
+
// so it never returned and held the owner-mutation fence for the whole drain, refusing every
|
|
93
|
+
// later deploy for hours (TensorBuzz production, 2026-11-16). The bound is far below the
|
|
94
|
+
// worker's own 30 000 ms stop delay because the stop must still let the generation's other
|
|
95
|
+
// processes exit; what it must not do is wait for the worker at all.
|
|
96
|
+
expect(Date.now() - startedAtMs).toBeLessThan(10000)
|
|
97
|
+
|
|
98
|
+
const stopped = statusRelease(daemon, "v1")
|
|
99
|
+
const stoppedWorker = stopped.processes.find((processStatus) => processStatus.id === "worker")
|
|
100
|
+
|
|
101
|
+
expect(stopped.state).toBe("stopped")
|
|
102
|
+
expect(stoppedWorker?.retirementTimedOut).toBe(true)
|
|
103
|
+
// The tracked record is the shell wrapper, which is already gone; the worker it launched is
|
|
104
|
+
// still draining inside the same process group, and that is the residue the stop must report.
|
|
105
|
+
expect(stoppedWorker?.heldProcessGroup?.liveMemberCount).toBeGreaterThan(0)
|
|
106
|
+
expect(logs.filter((entry) => entry.message === "release stop gave up waiting for a retiring process")).toMatchObject([
|
|
107
|
+
{data: {processId: "worker", releaseId: "v1", timeoutMs: 100}}
|
|
108
|
+
])
|
|
109
|
+
expect(logs.filter((entry) => entry.message === "release stopped")).toMatchObject([{data: {releaseId: "v1"}}])
|
|
110
|
+
} finally {
|
|
111
|
+
await daemon.shutdown()
|
|
112
|
+
if (workerPid !== undefined) {
|
|
113
|
+
try { process.kill(-workerPid, "SIGKILL") } catch (_error) { /* The group already exited. */ }
|
|
114
|
+
}
|
|
115
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
116
|
+
}
|
|
117
|
+
})
|
|
118
|
+
|
|
64
119
|
test("deploy switches new HTTP traffic while old WebSockets drain", async () => {
|
|
65
120
|
const fixture = await createFixture()
|
|
66
121
|
const daemon = await startDaemon(fixture.config)
|
|
@@ -2064,6 +2119,51 @@ test("a draining-generation bound of 0 leaves concurrent drains unbounded", asyn
|
|
|
2064
2119
|
}
|
|
2065
2120
|
})
|
|
2066
2121
|
|
|
2122
|
+
test("a deploy is the event that asks a retiring generation for its escalation decision, and nothing else does", async () => {
|
|
2123
|
+
const config = normalizeConfig({
|
|
2124
|
+
application: "demo",
|
|
2125
|
+
control: {path: path.join(os.tmpdir(), "rollbridge-escalate-gate.sock")},
|
|
2126
|
+
processes: [{command: `${JSON.stringify(process.execPath)} -e "setInterval(() => {}, 1000)"`, id: "web", policy: "proxied", port: {from: 0, to: 0}}],
|
|
2127
|
+
proxy: {host: "127.0.0.1", port: 0}
|
|
2128
|
+
})
|
|
2129
|
+
const daemon = new RollbridgeDaemon({config, logger: () => {}})
|
|
2130
|
+
/** @type {Array<{reason: string, releaseId: string}>} */
|
|
2131
|
+
const decided = []
|
|
2132
|
+
|
|
2133
|
+
/**
|
|
2134
|
+
* @param {string} releaseId - Generation id.
|
|
2135
|
+
* @param {import("../src/release-group.js").ReleaseState} state - Recorded state of the generation.
|
|
2136
|
+
* @returns {import("../src/release-group.js").default} A generation that only records the decision request.
|
|
2137
|
+
*/
|
|
2138
|
+
// eslint-disable-next-line local/no-unknown-jsdoc-type
|
|
2139
|
+
const retiringRelease = (releaseId, state) => /** @type {import("../src/release-group.js").default} */ (/** @type {unknown} */ ({
|
|
2140
|
+
escalateWedgedGeneration: async (/** @type {{reason: string}} */ options) => {
|
|
2141
|
+
decided.push({reason: options.reason, releaseId})
|
|
2142
|
+
|
|
2143
|
+
return []
|
|
2144
|
+
},
|
|
2145
|
+
releaseId,
|
|
2146
|
+
state
|
|
2147
|
+
}))
|
|
2148
|
+
|
|
2149
|
+
daemon.releases.set("draining", retiringRelease("draining", "draining"))
|
|
2150
|
+
daemon.releases.set("drained", retiringRelease("drained", "stopped"))
|
|
2151
|
+
daemon.releases.set("active", retiringRelease("active", "active"))
|
|
2152
|
+
|
|
2153
|
+
// Off by default: enabling the escalation is the operator's decision, not the daemon's.
|
|
2154
|
+
await daemon.escalateWedgedGenerations(config, "candidate")
|
|
2155
|
+
|
|
2156
|
+
expect(decided).toEqual([])
|
|
2157
|
+
|
|
2158
|
+
const escalating = normalizeConfig({...config, releaseRetention: {...config.releaseRetention, escalateWedgedGenerations: true}})
|
|
2159
|
+
|
|
2160
|
+
await daemon.escalateWedgedGenerations(escalating, "candidate")
|
|
2161
|
+
|
|
2162
|
+
// Every generation still holding a retirement — draining, drained with residue, or failed —
|
|
2163
|
+
// is asked; the active one and the candidate itself are not.
|
|
2164
|
+
expect(decided).toEqual([{reason: "deploy of candidate", releaseId: "draining"}, {reason: "deploy of candidate", releaseId: "drained"}])
|
|
2165
|
+
})
|
|
2166
|
+
|
|
2067
2167
|
test("status reports the drain classification of each draining generation", async () => {
|
|
2068
2168
|
const fixture = await createFixture()
|
|
2069
2169
|
const daemon = await startDaemon(fixture.config)
|
|
@@ -2127,6 +2227,79 @@ test("rollback refuses to restore a drained generation that still holds live pro
|
|
|
2127
2227
|
}
|
|
2128
2228
|
})
|
|
2129
2229
|
|
|
2230
|
+
test("deploy refuses to activate a release id whose draining generation still holds live processes", async () => {
|
|
2231
|
+
const fixture = await createFixture()
|
|
2232
|
+
const daemon = await startDaemon(fixture.config)
|
|
2233
|
+
/** @type {WebSocket | undefined} */
|
|
2234
|
+
let socket
|
|
2235
|
+
|
|
2236
|
+
try {
|
|
2237
|
+
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
2238
|
+
|
|
2239
|
+
// A held WebSocket keeps v1 draining after v2 takes traffic over, so v1's generation is
|
|
2240
|
+
// still alive when a restore names its id — the measured 2026-11-15 20:29Z and 20:41Z
|
|
2241
|
+
// shape, where the re-activation aborted with a bare guardian provenance mismatch.
|
|
2242
|
+
socket = await openWebSocket(daemon)
|
|
2243
|
+
await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
2244
|
+
|
|
2245
|
+
const draining = statusRelease(daemon, "v1")
|
|
2246
|
+
const heldPid = draining.processes.find((processStatus) => processStatus.id === "web")?.pid
|
|
2247
|
+
|
|
2248
|
+
if (!heldPid) throw new Error("the draining generation should still hold its web process")
|
|
2249
|
+
|
|
2250
|
+
await expect(daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})).rejects.toThrow(/Deploy refused before activation: release "v1" is retained in state draining while 1 process of that generation still hold live work \(web: state running, pid \d+\)\. Activating that release id again would re-register those processes under a new provenance and abort with a guardian provenance mismatch\. The guardian's active release is "v2"; deploy a new release id or reconcile the deployment pointer with `rollbridge status`\./)
|
|
2251
|
+
|
|
2252
|
+
const afterRefusal = daemon.status()
|
|
2253
|
+
|
|
2254
|
+
// Refusing before activation leaves one generation per release id: nothing was started,
|
|
2255
|
+
// the incumbent keeps serving, and the draining generation is still the one status reports.
|
|
2256
|
+
expect(afterRefusal.activeReleaseId).toBe("v2")
|
|
2257
|
+
expect(afterRefusal.releases.filter((release) => release.releaseId === "v1").length).toBe(1)
|
|
2258
|
+
expect(statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "web")?.pid).toBe(heldPid)
|
|
2259
|
+
expect(await fetchText(daemon, "/release")).toBe("v2")
|
|
2260
|
+
} finally {
|
|
2261
|
+
if (socket) socket.close()
|
|
2262
|
+
await daemon.shutdown()
|
|
2263
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
2264
|
+
}
|
|
2265
|
+
})
|
|
2266
|
+
|
|
2267
|
+
test("a restore deploy naming a drained generation that outlived its tracked process is refused by name", async () => {
|
|
2268
|
+
const fixture = await createFixture({nonBlockingDrainWorker: true, workerStopDelayMs: 6000})
|
|
2269
|
+
const config = normalizeConfig({
|
|
2270
|
+
...fixture.config,
|
|
2271
|
+
processes: fixture.config.processes.map((processConfig) => processConfig.id === "worker"
|
|
2272
|
+
? {...processConfig, gracefulStopMs: "indefinite"}
|
|
2273
|
+
: processConfig),
|
|
2274
|
+
proxy: {...fixture.config.proxy, nonBlockingDrainTimeoutMs: 200}
|
|
2275
|
+
})
|
|
2276
|
+
const daemon = await startDaemon(config)
|
|
2277
|
+
|
|
2278
|
+
try {
|
|
2279
|
+
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
2280
|
+
await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
2281
|
+
|
|
2282
|
+
// v1 is recorded as drained while its worker keeps its in-flight work in the group its
|
|
2283
|
+
// shell wrapper led. The 2026-10-25 carve-out that lets a dead generation's process key be
|
|
2284
|
+
// re-registered does not apply here: the group is alive, so the key is still held.
|
|
2285
|
+
await waitFor(() => statusRelease(daemon, "v1").state === "stopped")
|
|
2286
|
+
|
|
2287
|
+
const worker = statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")
|
|
2288
|
+
|
|
2289
|
+
if (!worker?.heldProcessGroup) {
|
|
2290
|
+
throw new Error(`the drained generation should still hold the worker's process group, got ${JSON.stringify(worker && {pid: worker.pid, retirementTimedOut: worker.retirementTimedOut, state: worker.state})}`)
|
|
2291
|
+
}
|
|
2292
|
+
|
|
2293
|
+
await expect(daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})).rejects.toThrow(/Deploy refused before activation: release "v1" is retained in state stopped while 1 process of that generation still hold live work \(worker: process group \d+ with \d+ live member\(s\) still alive after the tracked process exited\)\. Activating that release id again would re-register those processes under a new provenance and abort with a guardian provenance mismatch\. The guardian's active release is "v2"/)
|
|
2294
|
+
expect(daemon.status().activeReleaseId).toBe("v2")
|
|
2295
|
+
expect(daemon.status().releases.filter((release) => release.releaseId === "v1").length).toBe(1)
|
|
2296
|
+
expect(await fetchText(daemon, "/release")).toBe("v2")
|
|
2297
|
+
} finally {
|
|
2298
|
+
await daemon.shutdown()
|
|
2299
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
2300
|
+
}
|
|
2301
|
+
})
|
|
2302
|
+
|
|
2130
2303
|
test("the rollback control command switches traffic over the socket", async () => {
|
|
2131
2304
|
const fixture = await createFixture()
|
|
2132
2305
|
const daemon = await startDaemon(fixture.config)
|