rollbridge 0.1.79 → 0.1.81
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/test/managed-process.test.js +2 -237
package/package.json
CHANGED
|
@@ -1,13 +1,11 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
3
|
import fs from "node:fs"
|
|
4
|
-
import {spawn} from "node:child_process"
|
|
5
4
|
import os from "node:os"
|
|
6
5
|
import path from "node:path"
|
|
7
6
|
import {describe, expect, test} from "@velocious/testing"
|
|
8
7
|
import {fileURLToPath} from "node:url"
|
|
9
8
|
import ManagedProcess, {childRecordIsGone} from "../src/managed-process.js"
|
|
10
|
-
import {processStartTime} from "../src/process-memory.js"
|
|
11
9
|
|
|
12
10
|
describe("managed-process", () => {
|
|
13
11
|
|
|
@@ -113,27 +111,6 @@ test("reports zeroed restart and uptime fields before the process starts", () =>
|
|
|
113
111
|
expect(status.state).toBe("stopped")
|
|
114
112
|
})
|
|
115
113
|
|
|
116
|
-
test("re-adopts a live process whose supervisor status was stopped without spawning a duplicate", async () => {
|
|
117
|
-
const live = spawn(process.execPath, ["-e", "setTimeout(() => {}, 30000)"], {detached: true, stdio: "ignore"})
|
|
118
|
-
await new Promise((resolve, reject) => {
|
|
119
|
-
live.once("error", reject)
|
|
120
|
-
live.once("spawn", resolve)
|
|
121
|
-
})
|
|
122
|
-
if (!live.pid) throw new Error("Missing live process PID")
|
|
123
|
-
const startTime = processStartTime(live.pid)
|
|
124
|
-
if (!startTime) throw new Error("Missing live process start time")
|
|
125
|
-
const managed = buildProcess(10)
|
|
126
|
-
|
|
127
|
-
try {
|
|
128
|
-
expect(managed.status().state).toBe("stopped")
|
|
129
|
-
await managed.adopt({pid: live.pid, processStartTime: startTime}, "committed-config-digest")
|
|
130
|
-
|
|
131
|
-
expect(managed.status()).toMatchObject({pid: live.pid, processStartTime: startTime, state: "running"})
|
|
132
|
-
} finally {
|
|
133
|
-
await managed.stop()
|
|
134
|
-
}
|
|
135
|
-
})
|
|
136
|
-
|
|
137
114
|
test("counts automatic restarts and reports startedAt and uptime while running", async () => {
|
|
138
115
|
const managed = new ManagedProcess({
|
|
139
116
|
command: `${JSON.stringify(process.execPath)} ${JSON.stringify(crasherPath)}`,
|
|
@@ -332,12 +309,10 @@ test("runs quiet and drain lifecycle hooks before stopping", async () => {
|
|
|
332
309
|
})
|
|
333
310
|
|
|
334
311
|
test("retirement timeout leaves an indefinite worker alive without SIGKILL", async () => {
|
|
335
|
-
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-retirement-wedge-"))
|
|
336
|
-
const readyPath = path.join(dir, "ready")
|
|
337
312
|
/** @type {string[]} */
|
|
338
313
|
const signals = []
|
|
339
314
|
const managed = new ManagedProcess({
|
|
340
|
-
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(
|
|
315
|
+
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify("process.stdout.write('ready\\n'); process.on('SIGTERM', () => {}); setInterval(() => {}, 1000)")}`,
|
|
341
316
|
cwd: undefined,
|
|
342
317
|
env: {},
|
|
343
318
|
id: "worker",
|
|
@@ -357,218 +332,9 @@ test("retirement timeout leaves an indefinite worker alive without SIGKILL", asy
|
|
|
357
332
|
|
|
358
333
|
try {
|
|
359
334
|
await managed.start()
|
|
360
|
-
|
|
361
|
-
// child that has not finished booting reaches the default disposition instead, which exits
|
|
362
|
-
// the group and leaves no wedge to observe.
|
|
363
|
-
await waitFor(() => fs.existsSync(readyPath))
|
|
364
|
-
await managed.stop({retirementTimeoutMs: 25})
|
|
365
|
-
|
|
366
|
-
expect(managed.retirementTimedOut).toBe(true)
|
|
367
|
-
expect(signals).toEqual(["SIGTERM"])
|
|
368
|
-
expect(managed.status().pid).toBeDefined()
|
|
369
|
-
} finally {
|
|
370
|
-
const pid = managed.status().pid
|
|
371
|
-
if (pid) process.kill(-pid, "SIGKILL")
|
|
372
|
-
if (managed.exitPromise) await managed.exitPromise
|
|
373
|
-
fs.rmSync(dir, {force: true, recursive: true})
|
|
374
|
-
}
|
|
375
|
-
})
|
|
376
|
-
|
|
377
|
-
test("escalateRetirement terminates the wedged worker the bounded retirement left alive", async () => {
|
|
378
|
-
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-escalation-wedge-"))
|
|
379
|
-
const readyPath = path.join(dir, "ready")
|
|
380
|
-
/** @type {string[]} */
|
|
381
|
-
const signals = []
|
|
382
|
-
/** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
|
|
383
|
-
const logs = []
|
|
384
|
-
const managed = new ManagedProcess({
|
|
385
|
-
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`process.on('SIGTERM', () => {}); require("node:fs").writeFileSync(${JSON.stringify(readyPath)}, "ready"); setInterval(() => {}, 1000)`)}`,
|
|
386
|
-
cwd: undefined,
|
|
387
|
-
env: {},
|
|
388
|
-
id: "worker",
|
|
389
|
-
lifecycle: {drainTimeoutMs: 0},
|
|
390
|
-
logger: (message, data) => logs.push({data, message}),
|
|
391
|
-
outputLines: 50,
|
|
392
|
-
restartDelayMs: 10,
|
|
393
|
-
shouldRestart: () => false,
|
|
394
|
-
stopSignal: "SIGTERM",
|
|
395
|
-
stopTimeoutMs: "indefinite"
|
|
396
|
-
})
|
|
397
|
-
const signalProcessGroup = managed.signalProcessGroup.bind(managed)
|
|
398
|
-
|
|
399
|
-
managed.signalProcessGroup = async (signal, pgid, deadline) => {
|
|
400
|
-
signals.push(signal)
|
|
401
|
-
await signalProcessGroup(signal, pgid, deadline)
|
|
402
|
-
}
|
|
403
|
-
|
|
404
|
-
try {
|
|
405
|
-
await managed.start()
|
|
406
|
-
await waitFor(() => fs.existsSync(readyPath))
|
|
407
|
-
await managed.stop({retirementTimeoutMs: 25})
|
|
408
|
-
const wedgedPid = managed.status().pid
|
|
409
|
-
|
|
410
|
-
expect(managed.retirementTimedOut).toBe(true)
|
|
411
|
-
expect(signals).toEqual(["SIGTERM"])
|
|
412
|
-
expect(wedgedPid).toBeDefined()
|
|
413
|
-
|
|
414
|
-
expect(await managed.escalateRetirement({reason: "spec: proven wedge"})).toBe(true)
|
|
415
|
-
expect(signals).toEqual(["SIGTERM", "SIGKILL"])
|
|
416
|
-
expect(logs.some((entry) => entry.message === "process retirement escalated to SIGKILL" && entry.data?.reason === "spec: proven wedge")).toBe(true)
|
|
417
|
-
expect(await managed.waitForProcessGroupExit(/** @type {number} */ (wedgedPid), Date.now() + 5000)).toBe(true)
|
|
418
|
-
if (managed.exitPromise) await managed.exitPromise
|
|
419
|
-
expect(managed.status().pid).toBeUndefined()
|
|
420
|
-
expect(managed.status().state).toBe("stopped")
|
|
421
|
-
} finally {
|
|
422
|
-
const pid = managed.status().pid
|
|
423
|
-
if (pid) process.kill(-pid, "SIGKILL")
|
|
424
|
-
if (managed.exitPromise) await managed.exitPromise
|
|
425
|
-
fs.rmSync(dir, {force: true, recursive: true})
|
|
426
|
-
}
|
|
427
|
-
})
|
|
428
|
-
|
|
429
|
-
test("escalateRetirement refuses a process that has not expired its retirement budget", async () => {
|
|
430
|
-
/** @type {string[]} */
|
|
431
|
-
const signals = []
|
|
432
|
-
/** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
|
|
433
|
-
const logs = []
|
|
434
|
-
const managed = new ManagedProcess({
|
|
435
|
-
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify("setInterval(() => {}, 1000)")}`,
|
|
436
|
-
cwd: undefined,
|
|
437
|
-
env: {},
|
|
438
|
-
id: "worker",
|
|
439
|
-
lifecycle: {drainTimeoutMs: 0},
|
|
440
|
-
logger: (message, data) => logs.push({data, message}),
|
|
441
|
-
outputLines: 50,
|
|
442
|
-
restartDelayMs: 10,
|
|
443
|
-
shouldRestart: () => false,
|
|
444
|
-
stopSignal: "SIGTERM",
|
|
445
|
-
stopTimeoutMs: "indefinite"
|
|
446
|
-
})
|
|
447
|
-
const signalProcessGroup = managed.signalProcessGroup.bind(managed)
|
|
448
|
-
|
|
449
|
-
managed.signalProcessGroup = async (signal, pgid, deadline) => {
|
|
450
|
-
signals.push(signal)
|
|
451
|
-
await signalProcessGroup(signal, pgid, deadline)
|
|
452
|
-
}
|
|
453
|
-
|
|
454
|
-
try {
|
|
455
|
-
await managed.start()
|
|
456
|
-
await managed.quiesce()
|
|
457
|
-
|
|
458
|
-
expect(managed.retirementTimedOut).toBe(false)
|
|
459
|
-
expect(await managed.escalateRetirement({reason: "spec"})).toBe(false)
|
|
460
|
-
expect(signals).toEqual([])
|
|
461
|
-
expect(logs.some((entry) => entry.message === "retirement escalation refused; process is not past its retirement budget")).toBe(true)
|
|
462
|
-
expect(managed.status().pid).toBeDefined()
|
|
463
|
-
} finally {
|
|
464
|
-
const pid = managed.status().pid
|
|
465
|
-
if (pid) process.kill(-pid, "SIGKILL")
|
|
466
|
-
if (managed.exitPromise) await managed.exitPromise
|
|
467
|
-
}
|
|
468
|
-
})
|
|
469
|
-
|
|
470
|
-
test("escalateRetirement is inert for a process that drained normally", async () => {
|
|
471
|
-
/** @type {string[]} */
|
|
472
|
-
const signals = []
|
|
473
|
-
const managed = new ManagedProcess({
|
|
474
|
-
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify("setInterval(() => {}, 1000)")}`,
|
|
475
|
-
cwd: undefined,
|
|
476
|
-
env: {},
|
|
477
|
-
id: "worker",
|
|
478
|
-
lifecycle: {drainTimeoutMs: 0},
|
|
479
|
-
logger: () => {},
|
|
480
|
-
outputLines: 50,
|
|
481
|
-
restartDelayMs: 10,
|
|
482
|
-
shouldRestart: () => false,
|
|
483
|
-
stopSignal: "SIGTERM",
|
|
484
|
-
stopTimeoutMs: 2000
|
|
485
|
-
})
|
|
486
|
-
const signalProcessGroup = managed.signalProcessGroup.bind(managed)
|
|
487
|
-
|
|
488
|
-
managed.signalProcessGroup = async (signal, pgid, deadline) => {
|
|
489
|
-
signals.push(signal)
|
|
490
|
-
await signalProcessGroup(signal, pgid, deadline)
|
|
491
|
-
}
|
|
492
|
-
|
|
493
|
-
try {
|
|
494
|
-
await managed.start()
|
|
495
|
-
await managed.stop()
|
|
496
|
-
|
|
497
|
-
expect(managed.status().state).toBe("stopped")
|
|
498
|
-
expect(await managed.escalateRetirement({reason: "spec"})).toBe(false)
|
|
499
|
-
expect(signals).toEqual(["SIGTERM"])
|
|
500
|
-
} finally {
|
|
501
|
-
const pid = managed.status().pid
|
|
502
|
-
if (pid) process.kill(-pid, "SIGKILL")
|
|
503
|
-
if (managed.exitPromise) await managed.exitPromise
|
|
504
|
-
}
|
|
505
|
-
})
|
|
506
|
-
|
|
507
|
-
test("a retirement budget caps a finite graceful window instead of being outvoted by it", async () => {
|
|
508
|
-
/** @type {string[]} */
|
|
509
|
-
const signals = []
|
|
510
|
-
/** @type {Array<{msFromStopStart: number, phase: string}>} */
|
|
511
|
-
const deadlines = []
|
|
512
|
-
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-retirement-bound-"))
|
|
513
|
-
const readyPath = path.join(dir, "ready")
|
|
514
|
-
const managed = new ManagedProcess({
|
|
515
|
-
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`process.on('SIGTERM', () => {}); require("node:fs").writeFileSync(${JSON.stringify(readyPath)}, "ready"); setInterval(() => {}, 1000)`)}`,
|
|
516
|
-
cwd: undefined,
|
|
517
|
-
env: {},
|
|
518
|
-
id: "worker",
|
|
519
|
-
lifecycle: {drainTimeoutMs: 0},
|
|
520
|
-
logger: () => {},
|
|
521
|
-
outputLines: 50,
|
|
522
|
-
restartDelayMs: 10,
|
|
523
|
-
shouldRestart: () => false,
|
|
524
|
-
stopSignal: "SIGTERM",
|
|
525
|
-
stopTimeoutMs: 2000
|
|
526
|
-
})
|
|
527
|
-
let stopStartedAtMs = 0
|
|
528
|
-
|
|
529
|
-
/**
|
|
530
|
-
* @param {string} phase - The step the deadline was handed to.
|
|
531
|
-
* @param {number | undefined} deadline - Absolute deadline the step received.
|
|
532
|
-
* @returns {void} Records the deadline as an offset from the start of the stop.
|
|
533
|
-
*/
|
|
534
|
-
const recordDeadline = (phase, deadline) => {
|
|
535
|
-
if (deadline !== undefined) deadlines.push({msFromStopStart: deadline - stopStartedAtMs, phase})
|
|
536
|
-
}
|
|
537
|
-
|
|
538
|
-
const signalProcessGroup = managed.signalProcessGroup.bind(managed)
|
|
539
|
-
managed.signalProcessGroup = async (signal, pgid, deadline) => {
|
|
540
|
-
signals.push(signal)
|
|
541
|
-
recordDeadline("signal", deadline)
|
|
542
|
-
await signalProcessGroup(signal, pgid, deadline)
|
|
543
|
-
}
|
|
544
|
-
|
|
545
|
-
const waitForProcessGroupExit = managed.waitForProcessGroupExit.bind(managed)
|
|
546
|
-
managed.waitForProcessGroupExit = async (pgid, deadline) => {
|
|
547
|
-
recordDeadline("wait", deadline)
|
|
548
|
-
return await waitForProcessGroupExit(pgid, deadline)
|
|
549
|
-
}
|
|
550
|
-
|
|
551
|
-
try {
|
|
552
|
-
await managed.start()
|
|
553
|
-
// The fixture installs its SIGTERM disposition before it announces readiness. Signalling a
|
|
554
|
-
// child that has not finished booting reaches the default disposition instead, which exits
|
|
555
|
-
// the group and leaves no retirement wait to observe.
|
|
556
|
-
await waitFor(() => fs.existsSync(readyPath))
|
|
557
|
-
stopStartedAtMs = Date.now()
|
|
558
|
-
|
|
335
|
+
await waitFor(() => managed.status().logs.some(({line}) => line === "ready"))
|
|
559
336
|
await managed.stop({retirementTimeoutMs: 25})
|
|
560
337
|
|
|
561
|
-
// The 2 000 ms graceful window must not outvote the 25 ms retirement budget. Read the other
|
|
562
|
-
// way round the wait runs the whole window before it escalates to SIGKILL, which is the shape
|
|
563
|
-
// that held the TensorBuzz control plane for 4,5 h after the worker moved from "indefinite" to
|
|
564
|
-
// 4,5 h and killed the in-flight builds the consumer said not to interrupt.
|
|
565
|
-
expect(deadlines.map((entry) => entry.phase)).toEqual(["signal", "wait"])
|
|
566
|
-
for (const entry of deadlines) {
|
|
567
|
-
// The deadline starts at stop entry; a millisecond clock tick between this test's marker
|
|
568
|
-
// and the async stop call may shave 1 ms from the observed offset.
|
|
569
|
-
expect(entry.msFromStopStart).toBeGreaterThanOrEqual(0)
|
|
570
|
-
expect(entry.msFromStopStart).toBeLessThan(1000)
|
|
571
|
-
}
|
|
572
338
|
expect(managed.retirementTimedOut).toBe(true)
|
|
573
339
|
expect(signals).toEqual(["SIGTERM"])
|
|
574
340
|
expect(managed.status().pid).toBeDefined()
|
|
@@ -576,7 +342,6 @@ test("a retirement budget caps a finite graceful window instead of being outvote
|
|
|
576
342
|
const pid = managed.status().pid
|
|
577
343
|
if (pid) process.kill(-pid, "SIGKILL")
|
|
578
344
|
if (managed.exitPromise) await managed.exitPromise
|
|
579
|
-
fs.rmSync(dir, {force: true, recursive: true})
|
|
580
345
|
}
|
|
581
346
|
})
|
|
582
347
|
|