rollbridge 0.1.73 → 0.1.75

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/README.md +63 -10
  2. package/TODO.md +13 -0
  3. package/changelog.d/20261025-guardian-re-register-dead-generation.md +13 -0
  4. package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +1 -0
  5. package/changelog.d/20261104-compose-service-units.md +21 -0
  6. package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +20 -0
  7. package/changelog.d/20261106-release-dependency-boot-budgets.md +32 -0
  8. package/changelog.d/20261109-path-filtered-builds.md +11 -0
  9. package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +26 -0
  10. package/changelog.d/20261111-contention-sensitive-daemon-specs.md +27 -0
  11. package/changelog.d/20261113-bounded-nonblocking-retirement.md +5 -0
  12. package/changelog.d/20261115-bound-draining-release-groups.md +62 -0
  13. package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +27 -0
  14. package/changelog.d/20261115-wedged-generation-escalation.md +44 -0
  15. package/changelog.d/20261116-bounded-explicit-stop-retirement.md +30 -0
  16. package/changelog.d/20261116-config-json-canonical-shape.md +11 -0
  17. package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +29 -0
  18. package/changelog.d/20261116-guardian-version-handoff.md +37 -0
  19. package/changelog.d/20261116-reclaim-dead-owner-mutation.md +7 -0
  20. package/changelog.d/20261117-activate-before-incumbent-retirement.md +1 -0
  21. package/changelog.d/20261117-activation-path-drain-deadline.md +35 -0
  22. package/changelog.d/20261117-stop-companion-after-retired-dependency.md +7 -0
  23. package/changelog.d/20261118-bounded-control-client-close.md +4 -0
  24. package/changelog.d/20261118-incumbent-retirement-authority-guard.md +1 -0
  25. package/changelog.d/20261118-owner-retirement-no-guardian-bound.md +19 -0
  26. package/changelog.d/20261118-shutdown-retirement-bound-restored.md +31 -0
  27. package/docs/cli.md +128 -3
  28. package/docs/compose-services.md +222 -0
  29. package/docs/config.md +177 -6
  30. package/docs/troubleshooting.md +97 -6
  31. package/docs/velocious.md +1 -1
  32. package/docs/workers.md +11 -0
  33. package/examples/tensorbuzz.com.js +3 -2
  34. package/package.json +1 -1
  35. package/src/cli.js +69 -3
  36. package/src/compose-unit.js +725 -0
  37. package/src/config.js +346 -8
  38. package/src/control-client.js +32 -5
  39. package/src/daemon.js +448 -76
  40. package/src/doctor.js +74 -0
  41. package/src/drain-policy.js +252 -0
  42. package/src/guardian-client.js +92 -9
  43. package/src/health.js +99 -8
  44. package/src/managed-process.js +197 -16
  45. package/src/process-guardian.js +212 -22
  46. package/src/process-memory.js +62 -2
  47. package/src/process-order.js +26 -0
  48. package/src/release-group.js +452 -39
  49. package/tensorbuzz.yml +20 -1
  50. package/test/completion.test.js +5 -2
  51. package/test/compose-config.test.js +160 -0
  52. package/test/compose-docker.test.js +297 -0
  53. package/test/compose-unit.test.js +415 -0
  54. package/test/config-validation.test.js +135 -5
  55. package/test/control-protocol.test.js +22 -0
  56. package/test/drain-policy.test.js +328 -0
  57. package/test/fixtures/deferred-boot-app.js +106 -0
  58. package/test/fixtures/gated-app.js +50 -0
  59. package/test/guardian-client.test.js +149 -5
  60. package/test/health.test.js +74 -1
  61. package/test/managed-process.test.js +260 -1
  62. package/test/owner-recovery.test.js +151 -20
  63. package/test/owner-replacement.test.js +176 -15
  64. package/test/process-memory.test.js +47 -1
  65. package/test/release-group.test.js +447 -2
  66. package/test/rollbridge.test.js +774 -72
  67. package/test/shutdown-completion.test.js +119 -0
  68. package/test/shutdown-retirement-bound.test.js +162 -0
  69. package/test/support/process.js +3 -1
  70. package/test/tensorbuzz-config.test.js +228 -0
@@ -7,19 +7,39 @@ import os from "node:os"
7
7
  import path from "node:path"
8
8
  import {describe, expect, test} from "@velocious/testing"
9
9
  import {fileURLToPath} from "node:url"
10
- import GuardianClient from "../src/guardian-client.js"
10
+ import {createRequire} from "node:module"
11
+ import GuardianClient, {usableSocketPath} from "../src/guardian-client.js"
11
12
  import {waitForProcessExit} from "./support/process.js"
12
13
 
13
14
  describe("guardian-client", () => {
14
15
 
16
+ const commonJsRequire = createRequire(import.meta.url)
17
+ const packageVersion = /** @type {{version: string}} */ (commonJsRequire("../package.json")).version
15
18
  const legacyGuardianPath = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures", "pre-split3-process-guardian.js")
16
19
  const recoveryOwnerPath = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures", "guardian-recovery-owner.js")
17
20
 
21
+ test("guardian maps an overlong configured socket path to a stable short path", async () => {
22
+ const root = await fs.mkdtemp(path.join(os.tmpdir(), `rollbridge-${"x".repeat(120)}-`))
23
+ const configuredPath = path.join(root, "guardian.sock")
24
+ const client = new GuardianClient({socketPath: configuredPath, token: "long-path-capability"})
25
+
26
+ try {
27
+ expect(Buffer.byteLength(configuredPath)).toBeGreaterThan(107)
28
+ expect(Buffer.byteLength(client.socketPath)).toBeLessThanOrEqual(107)
29
+ expect(client.socketPath).not.toBe(configuredPath)
30
+ await client.launch()
31
+ expect(await client.capabilities()).toEqual({daemonRecovery: 1, generationReactivation: 1, guardianVersion: packageVersion})
32
+ } finally {
33
+ await cleanupGuardian({client, root})
34
+ }
35
+ })
36
+
18
37
  test("guardian bootstrap capability is absent from process argv", async () => {
19
38
  const fixture = await createGuardian()
39
+ const packageVersion = JSON.parse(await fs.readFile(path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "package.json"), "utf8")).version
20
40
 
21
41
  try {
22
- expect(await fixture.client.capabilities()).toEqual({daemonRecovery: 1, generationReactivation: 1})
42
+ expect(await fixture.client.capabilities()).toEqual({daemonRecovery: 1, generationReactivation: 1, guardianVersion: packageVersion})
23
43
  const commandLine = await fs.readFile(`/proc/${fixture.client.pid}/cmdline`, "utf8")
24
44
  const environment = await fs.readFile(`/proc/${fixture.client.pid}/environ`, "utf8")
25
45
  const status = await fs.readFile(`/proc/${fixture.client.pid}/status`, "utf8")
@@ -58,6 +78,100 @@ test("guardian inventory removes only an exact owned provenance", async () => {
58
78
  }
59
79
  })
60
80
 
81
+ test("a refused owner mutation names the held mutation, its process, its age and its blocker", async () => {
82
+ const fixture = await createGuardian()
83
+
84
+ try {
85
+ const mutationId = await fixture.client.beginOwnerMutation("restart", {ownerPid: process.pid, waitingFor: "the managed process to stop and start again"})
86
+ await expect(fixture.client.beginOwnerMutation("deploy")).rejects.toThrow(/^Another owner mutation is already in progress: owner mutation "restart"; mutation [0-9a-f-]{36}; held by process \d+; since 20\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\.\d\d\dZ; \d+s elapsed; waiting for the managed process to stop and start again; refused deploy$/)
87
+ await fixture.client.reportOwnerMutationWait(mutationId, "an idle forked runner that never exits")
88
+ await expect(fixture.client.beginOwnerMutation("deploy")).rejects.toThrow(/waiting for an idle forked runner that never exits/)
89
+ await fixture.client.endOwnerMutation(mutationId)
90
+ expect(typeof await fixture.client.beginOwnerMutation("deploy")).toBe("string")
91
+ } finally {
92
+ await cleanupGuardian(fixture)
93
+ }
94
+ })
95
+
96
+ test("an owner mutation that outruns its lease is released and the holder reports the expiry", async () => {
97
+ const fixture = await createGuardian()
98
+
99
+ try {
100
+ const wedged = await fixture.client.beginOwnerMutation("restart", {mutationTimeoutMs: 200, ownerPid: process.pid, waitingFor: "a drain that never returns"})
101
+
102
+ await new Promise((resolve) => setTimeout(resolve, 600))
103
+ const successor = await fixture.client.beginOwnerMutation("deploy", {ownerPid: process.pid, waitingFor: "candidate activation"})
104
+
105
+ expect(typeof successor).toBe("string")
106
+ await expect(fixture.client.endOwnerMutation(wedged)).rejects.toThrow(/^Owner mutation restart exceeded its 200 ms lease and was released after \d+ ms; the mutation failed and its lease is gone$/)
107
+ await fixture.client.endOwnerMutation(successor)
108
+ } finally {
109
+ await cleanupGuardian(fixture)
110
+ }
111
+ })
112
+
113
+ test("a dead owner-mutation holder is reclaimed and the next mutation is admitted", async () => {
114
+ const fixture = await createGuardian()
115
+ const holder = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"])
116
+
117
+ try {
118
+ const reclaimed = fixture.client.waitForEvent("owner-mutation-reclaimed")
119
+ await fixture.client.beginOwnerMutation("restart", {ownerPid: holder.pid, waitingFor: "a process that will die"})
120
+ if (!holder.pid) throw new Error("Missing mutation holder PID")
121
+ holder.kill("SIGKILL")
122
+ await once(holder, "exit")
123
+
124
+ const successor = await fixture.client.beginOwnerMutation("deploy", {ownerPid: process.pid, waitingFor: "candidate activation"})
125
+ const event = await reclaimed
126
+
127
+ expect(typeof successor).toBe("string")
128
+ expect(event).toEqual(expect.objectContaining({code: "OWNER_MUTATION_RECLAIMED", event: "owner-mutation-reclaimed", operation: "restart", pid: holder.pid, reason: "owner mutation holder process is no longer alive"}))
129
+ expect((await fixture.client.guardianStatus()).version).toBe(packageVersion)
130
+ await fixture.client.endOwnerMutation(successor)
131
+ } finally {
132
+ if (holder.exitCode === null && holder.signalCode === null) holder.kill("SIGKILL")
133
+ await cleanupGuardian(fixture)
134
+ }
135
+ })
136
+
137
+ test("guardian lets the owner re-register a key whose registered child is gone, and still fences a live one", async () => {
138
+ const fixture = await createGuardian()
139
+ const liveInstance = fixture.client.process("live-generation", definition("live-generation"))
140
+ const deadInstance = fixture.client.process("dead-generation", definition("dead-generation"))
141
+
142
+ try {
143
+ await liveInstance.start()
144
+ await deadInstance.start()
145
+ const deadPid = (await deadInstance.status()).pid
146
+
147
+ if (!deadPid) throw new Error("Missing required fixture: dead-generation pid")
148
+ await deadInstance.stop()
149
+ await waitForProcessExit(deadPid)
150
+ const deadStatus = await deadInstance.status()
151
+
152
+ expect(deadStatus.state).toBe("stopped")
153
+ expect(deadStatus.pid).toBeUndefined()
154
+
155
+ // The wedge this guards against: a later deploy reconstructs the same key from a
156
+ // different release definition. The old child is conclusively gone, so the old
157
+ // provenance must not fence the new registration.
158
+ const nextInstance = fixture.client.process("dead-generation", {...definition("dead-generation"), env: {REPLACED_BY_LATER_DEPLOY: "1"}})
159
+ const reRegistered = nextInstance.status()
160
+
161
+ expect(reRegistered.state).toBe("stopped")
162
+ await nextInstance.start()
163
+ expect((await nextInstance.status()).state).toBe("running")
164
+ expect((await fixture.client.inventory()).some((entry) => entry.key === "dead-generation" && entry.provenance === nextInstance.provenance)).toBe(true)
165
+
166
+ // A live process with a different provenance stays fail-closed.
167
+ const hostileInstance = fixture.client.process("live-generation", {...definition("live-generation"), env: {HOSTILE: "1"}})
168
+
169
+ await expect(hostileInstance.start()).rejects.toThrow(/provenance mismatch/)
170
+ } finally {
171
+ await cleanupGuardian(fixture)
172
+ }
173
+ })
174
+
61
175
  test("guardian runs a strict activation lifecycle command for the exact registered process", async () => {
62
176
  const fixture = await createGuardian()
63
177
  const activationPath = path.join(fixture.root, "activated")
@@ -842,7 +956,7 @@ test("replacement staging rejects owner state published after prepare", async ()
842
956
  const prepared = await candidate.prepareOwnerReplacement(authority, nextAuthority)
843
957
 
844
958
  await fixture.client.publishOwnerState({authority, snapshot: {activeReleaseId: "v2"}})
845
- await expect(candidate.stageOwnerReplacement(prepared.replacementId, {authority: nextAuthority, snapshot: {activeReleaseId: "v1"}})).rejects.toThrow(/owner state changed after prepare/i)
959
+ await expect(candidate.stageOwnerReplacement(prepared.replacementId, {authority: nextAuthority, snapshot: {activeReleaseId: "v1"}})).rejects.toThrow(/owner state changed after prepare \(prepared revision 1, current revision 2\)/i)
846
960
  await candidate.abortOwnerReplacement(prepared.replacementId)
847
961
  const fresh = await candidate.prepareOwnerReplacement(authority, nextAuthority)
848
962
 
@@ -1527,6 +1641,36 @@ test("first upgrade migrates a real pre-split guardian without replacing its own
1527
1641
  }
1528
1642
  })
1529
1643
 
1644
+ test("a current client re-adopts a stopped record from a legacy guardian", async () => {
1645
+ const fixture = await createLegacyGuardian()
1646
+ const processDefinition = definition("legacy-restore")
1647
+ const legacyProcess = fixture.client.process("release:v1:legacy-restore", processDefinition)
1648
+ let replacement
1649
+
1650
+ try {
1651
+ await legacyProcess.start()
1652
+ const pid = legacyProcess.status().pid
1653
+
1654
+ if (!pid) throw new Error("Missing required legacy process pid")
1655
+ await legacyProcess.stop()
1656
+ await waitForProcessExit(pid)
1657
+
1658
+ replacement = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.client.token})
1659
+ await replacement.connect()
1660
+ const restored = replacement.process("release:v1:legacy-restore", {...processDefinition, env: {RESTORED_BY_CURRENT_CLIENT: "1"}})
1661
+
1662
+ await restored.recover()
1663
+ await restored.start()
1664
+ expect(restored.status().state).toBe("running")
1665
+ } finally {
1666
+ await replacement?.shutdown().catch(() => {})
1667
+ replacement?.disconnect()
1668
+ fixture.client.disconnect()
1669
+ if (fixture.child.exitCode === null && fixture.child.signalCode === null) fixture.child.kill("SIGKILL")
1670
+ await fs.rm(fixture.root, {force: true, recursive: true})
1671
+ }
1672
+ })
1673
+
1530
1674
  test("split guardian rejects an owner-state update when its nested legacy definition update fails", async () => {
1531
1675
  const fixture = await createLegacyGuardian()
1532
1676
  const processDefinition = definition("legacy-worker")
@@ -1678,7 +1822,7 @@ test("a legacy bridge remains discoverable when its candidate disconnects after
1678
1822
  let replacement
1679
1823
 
1680
1824
  try {
1681
- const identity = {socketPath: path.join(fixture.root, "guardian-v2.sock"), token: "candidate-guardian-capability"}
1825
+ const identity = {socketPath: usableSocketPath(path.join(fixture.root, "guardian-v2.sock")), token: "candidate-guardian-capability"}
1682
1826
 
1683
1827
  upgraded = await fixture.client.upgradeLegacyGuardian({ownerState, ...identity})
1684
1828
  const prepared = await upgraded.prepareOwnerReplacement(authority, nextAuthority)
@@ -1725,7 +1869,7 @@ async function createGuardian() {
1725
1869
  async function createLegacyGuardian() {
1726
1870
  const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-legacy-guardian-"))
1727
1871
  const token = "legacy-guardian-capability"
1728
- const socketPath = path.join(root, "guardian.sock")
1872
+ const socketPath = usableSocketPath(path.join(root, "guardian.sock"))
1729
1873
  const child = spawn(process.execPath, [legacyGuardianPath, socketPath], {detached: true, stdio: ["ignore", "ignore", "ignore", "ipc"]})
1730
1874
 
1731
1875
  await new Promise((resolve, reject) => {
@@ -1,8 +1,9 @@
1
1
  // @ts-check
2
2
 
3
3
  import http from "node:http"
4
+ import net from "node:net"
4
5
  import {describe, expect, test} from "@velocious/testing"
5
- import {waitForHealth} from "../src/health.js"
6
+ import {waitForHealth, waitForListeningPort} from "../src/health.js"
6
7
 
7
8
  describe("health", () => {
8
9
 
@@ -31,6 +32,18 @@ async function startHealthServer() {
31
32
  }
32
33
  }
33
34
 
35
+ /**
36
+ * @param {net.Server} server - Server to bind.
37
+ * @returns {Promise<number>} Ephemeral port it bound.
38
+ */
39
+ async function bindPort(server) {
40
+ await new Promise((resolve) => server.listen(0, "127.0.0.1", () => resolve(undefined)))
41
+
42
+ const address = server.address()
43
+
44
+ return address && typeof address === "object" ? address.port : 0
45
+ }
46
+
34
47
  test("waitForHealth delays the first probe by startDelayMs", async () => {
35
48
  const server = await startHealthServer()
36
49
 
@@ -62,4 +75,64 @@ test("waitForHealth probes immediately when startDelayMs is 0", async () => {
62
75
  await server.close()
63
76
  }
64
77
  })
78
+
79
+ test("waitForHealth names the deadline and the time it spent when the app never becomes healthy", async () => {
80
+ const server = http.createServer((_request, response) => {
81
+ response.writeHead(500)
82
+ response.end("bad")
83
+ })
84
+
85
+ try {
86
+ const port = await bindPort(server)
87
+
88
+ /** @type {import("../src/health.js").HealthCheckError} */
89
+ const failure = /** @type {import("../src/health.js").HealthCheckError} */ (await waitForHealth({
90
+ health: {intervalMs: 25, path: "/ping", startDelayMs: 0, timeoutMs: 300},
91
+ host: "127.0.0.1",
92
+ port,
93
+ processId: "web"
94
+ }).catch((error) => error instanceof Error ? error : new Error(String(error))))
95
+
96
+ expect(String(failure.message)).toMatch(/process web: \d+ probes, \d+ms elapsed of the 300ms health deadline/)
97
+ expect(failure.healthCheck?.deadlineMs).toBe(300)
98
+ expect({value: Boolean(failure.healthCheck && failure.healthCheck.elapsedMs >= 300), context: `expected the health gate to have spent its whole budget, was ${failure.healthCheck?.elapsedMs}ms`}).toMatchObject({value: true})
99
+ expect({value: Boolean(failure.healthCheck && failure.healthCheck.attempts >= 1), context: `expected at least one probe, was ${failure.healthCheck?.attempts}`}).toMatchObject({value: true})
100
+ expect(failure.healthCheck?.processId).toBe("web")
101
+ } finally {
102
+ await server.close()
103
+ }
104
+ })
105
+
106
+ test("waitForListeningPort waits for a dependency port on a budget of its own", async () => {
107
+ const server = net.createServer()
108
+ const port = await bindPort(server)
109
+
110
+ await new Promise((resolve) => server.close(() => resolve(undefined)))
111
+
112
+ const waiting = waitForListeningPort({host: "127.0.0.1", intervalMs: 25, port, timeoutMs: 2000})
113
+
114
+ await new Promise((resolve) => setTimeout(resolve, 300))
115
+ await new Promise((resolve) => server.listen(port, "127.0.0.1", () => resolve(undefined)))
116
+
117
+ try {
118
+ const result = await waiting
119
+
120
+ expect({value: Boolean(result.attempts >= 2), context: `expected more than one connection attempt, was ${result.attempts}`}).toMatchObject({value: true})
121
+ expect({value: Boolean(result.elapsedMs >= 250), context: `expected the dependency wait to take ~300ms, was ${result.elapsedMs}ms`}).toMatchObject({value: true})
122
+ } finally {
123
+ await new Promise((resolve) => server.close(() => resolve(undefined)))
124
+ }
125
+ })
126
+
127
+ test("waitForListeningPort names the budget it ran out on when the dependency never binds", async () => {
128
+ const server = net.createServer()
129
+ const port = await bindPort(server)
130
+
131
+ await new Promise((resolve) => server.close(() => resolve(undefined)))
132
+
133
+ /** @type {Error} */
134
+ const failure = /** @type {Error} */ (await waitForListeningPort({host: "127.0.0.1", intervalMs: 25, port, timeoutMs: 200}).catch((error) => error instanceof Error ? error : new Error(String(error))))
135
+
136
+ expect(String(failure.message)).toMatch(/did not accept a connection within 200ms/)
137
+ })
65
138
  })
@@ -6,7 +6,7 @@ import os from "node:os"
6
6
  import path from "node:path"
7
7
  import {describe, expect, test} from "@velocious/testing"
8
8
  import {fileURLToPath} from "node:url"
9
- import ManagedProcess from "../src/managed-process.js"
9
+ import ManagedProcess, {childRecordIsGone} from "../src/managed-process.js"
10
10
  import {processStartTime} from "../src/process-memory.js"
11
11
 
12
12
  describe("managed-process", () => {
@@ -331,6 +331,255 @@ test("runs quiet and drain lifecycle hooks before stopping", async () => {
331
331
  }
332
332
  })
333
333
 
334
+ test("retirement timeout leaves an indefinite worker alive without SIGKILL", async () => {
335
+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-retirement-wedge-"))
336
+ const readyPath = path.join(dir, "ready")
337
+ /** @type {string[]} */
338
+ const signals = []
339
+ const managed = new ManagedProcess({
340
+ command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`process.on('SIGTERM', () => {}); require("node:fs").writeFileSync(${JSON.stringify(readyPath)}, "ready"); setInterval(() => {}, 1000)`)}`,
341
+ cwd: undefined,
342
+ env: {},
343
+ id: "worker",
344
+ lifecycle: {drainTimeoutMs: 0},
345
+ logger: () => {},
346
+ outputLines: 50,
347
+ restartDelayMs: 10,
348
+ shouldRestart: () => false,
349
+ stopSignal: "SIGTERM",
350
+ stopTimeoutMs: "indefinite"
351
+ })
352
+ const signalProcessGroup = managed.signalProcessGroup.bind(managed)
353
+ managed.signalProcessGroup = async (signal, pgid, deadline) => {
354
+ signals.push(signal)
355
+ await signalProcessGroup(signal, pgid, deadline)
356
+ }
357
+
358
+ try {
359
+ await managed.start()
360
+ // The fixture installs its SIGTERM disposition before it announces readiness. Signalling a
361
+ // child that has not finished booting reaches the default disposition instead, which exits
362
+ // the group and leaves no wedge to observe.
363
+ await waitFor(() => fs.existsSync(readyPath))
364
+ await managed.stop({retirementTimeoutMs: 25})
365
+
366
+ expect(managed.retirementTimedOut).toBe(true)
367
+ expect(signals).toEqual(["SIGTERM"])
368
+ expect(managed.status().pid).toBeDefined()
369
+ } finally {
370
+ const pid = managed.status().pid
371
+ if (pid) process.kill(-pid, "SIGKILL")
372
+ if (managed.exitPromise) await managed.exitPromise
373
+ fs.rmSync(dir, {force: true, recursive: true})
374
+ }
375
+ })
376
+
377
+ test("escalateRetirement terminates the wedged worker the bounded retirement left alive", async () => {
378
+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-escalation-wedge-"))
379
+ const readyPath = path.join(dir, "ready")
380
+ /** @type {string[]} */
381
+ const signals = []
382
+ /** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
383
+ const logs = []
384
+ const managed = new ManagedProcess({
385
+ command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`process.on('SIGTERM', () => {}); require("node:fs").writeFileSync(${JSON.stringify(readyPath)}, "ready"); setInterval(() => {}, 1000)`)}`,
386
+ cwd: undefined,
387
+ env: {},
388
+ id: "worker",
389
+ lifecycle: {drainTimeoutMs: 0},
390
+ logger: (message, data) => logs.push({data, message}),
391
+ outputLines: 50,
392
+ restartDelayMs: 10,
393
+ shouldRestart: () => false,
394
+ stopSignal: "SIGTERM",
395
+ stopTimeoutMs: "indefinite"
396
+ })
397
+ const signalProcessGroup = managed.signalProcessGroup.bind(managed)
398
+
399
+ managed.signalProcessGroup = async (signal, pgid, deadline) => {
400
+ signals.push(signal)
401
+ await signalProcessGroup(signal, pgid, deadline)
402
+ }
403
+
404
+ try {
405
+ await managed.start()
406
+ await waitFor(() => fs.existsSync(readyPath))
407
+ await managed.stop({retirementTimeoutMs: 25})
408
+ const wedgedPid = managed.status().pid
409
+
410
+ expect(managed.retirementTimedOut).toBe(true)
411
+ expect(signals).toEqual(["SIGTERM"])
412
+ expect(wedgedPid).toBeDefined()
413
+
414
+ expect(await managed.escalateRetirement({reason: "spec: proven wedge"})).toBe(true)
415
+ expect(signals).toEqual(["SIGTERM", "SIGKILL"])
416
+ expect(logs.some((entry) => entry.message === "process retirement escalated to SIGKILL" && entry.data?.reason === "spec: proven wedge")).toBe(true)
417
+ expect(await managed.waitForProcessGroupExit(/** @type {number} */ (wedgedPid), Date.now() + 5000)).toBe(true)
418
+ if (managed.exitPromise) await managed.exitPromise
419
+ expect(managed.status().pid).toBeUndefined()
420
+ expect(managed.status().state).toBe("stopped")
421
+ } finally {
422
+ const pid = managed.status().pid
423
+ if (pid) process.kill(-pid, "SIGKILL")
424
+ if (managed.exitPromise) await managed.exitPromise
425
+ fs.rmSync(dir, {force: true, recursive: true})
426
+ }
427
+ })
428
+
429
+ test("escalateRetirement refuses a process that has not expired its retirement budget", async () => {
430
+ /** @type {string[]} */
431
+ const signals = []
432
+ /** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
433
+ const logs = []
434
+ const managed = new ManagedProcess({
435
+ command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify("setInterval(() => {}, 1000)")}`,
436
+ cwd: undefined,
437
+ env: {},
438
+ id: "worker",
439
+ lifecycle: {drainTimeoutMs: 0},
440
+ logger: (message, data) => logs.push({data, message}),
441
+ outputLines: 50,
442
+ restartDelayMs: 10,
443
+ shouldRestart: () => false,
444
+ stopSignal: "SIGTERM",
445
+ stopTimeoutMs: "indefinite"
446
+ })
447
+ const signalProcessGroup = managed.signalProcessGroup.bind(managed)
448
+
449
+ managed.signalProcessGroup = async (signal, pgid, deadline) => {
450
+ signals.push(signal)
451
+ await signalProcessGroup(signal, pgid, deadline)
452
+ }
453
+
454
+ try {
455
+ await managed.start()
456
+ await managed.quiesce()
457
+
458
+ expect(managed.retirementTimedOut).toBe(false)
459
+ expect(await managed.escalateRetirement({reason: "spec"})).toBe(false)
460
+ expect(signals).toEqual([])
461
+ expect(logs.some((entry) => entry.message === "retirement escalation refused; process is not past its retirement budget")).toBe(true)
462
+ expect(managed.status().pid).toBeDefined()
463
+ } finally {
464
+ const pid = managed.status().pid
465
+ if (pid) process.kill(-pid, "SIGKILL")
466
+ if (managed.exitPromise) await managed.exitPromise
467
+ }
468
+ })
469
+
470
+ test("escalateRetirement is inert for a process that drained normally", async () => {
471
+ /** @type {string[]} */
472
+ const signals = []
473
+ const managed = new ManagedProcess({
474
+ command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify("setInterval(() => {}, 1000)")}`,
475
+ cwd: undefined,
476
+ env: {},
477
+ id: "worker",
478
+ lifecycle: {drainTimeoutMs: 0},
479
+ logger: () => {},
480
+ outputLines: 50,
481
+ restartDelayMs: 10,
482
+ shouldRestart: () => false,
483
+ stopSignal: "SIGTERM",
484
+ stopTimeoutMs: 2000
485
+ })
486
+ const signalProcessGroup = managed.signalProcessGroup.bind(managed)
487
+
488
+ managed.signalProcessGroup = async (signal, pgid, deadline) => {
489
+ signals.push(signal)
490
+ await signalProcessGroup(signal, pgid, deadline)
491
+ }
492
+
493
+ try {
494
+ await managed.start()
495
+ await managed.stop()
496
+
497
+ expect(managed.status().state).toBe("stopped")
498
+ expect(await managed.escalateRetirement({reason: "spec"})).toBe(false)
499
+ expect(signals).toEqual(["SIGTERM"])
500
+ } finally {
501
+ const pid = managed.status().pid
502
+ if (pid) process.kill(-pid, "SIGKILL")
503
+ if (managed.exitPromise) await managed.exitPromise
504
+ }
505
+ })
506
+
507
+ test("a retirement budget caps a finite graceful window instead of being outvoted by it", async () => {
508
+ /** @type {string[]} */
509
+ const signals = []
510
+ /** @type {Array<{msFromStopStart: number, phase: string}>} */
511
+ const deadlines = []
512
+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-retirement-bound-"))
513
+ const readyPath = path.join(dir, "ready")
514
+ const managed = new ManagedProcess({
515
+ command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`process.on('SIGTERM', () => {}); require("node:fs").writeFileSync(${JSON.stringify(readyPath)}, "ready"); setInterval(() => {}, 1000)`)}`,
516
+ cwd: undefined,
517
+ env: {},
518
+ id: "worker",
519
+ lifecycle: {drainTimeoutMs: 0},
520
+ logger: () => {},
521
+ outputLines: 50,
522
+ restartDelayMs: 10,
523
+ shouldRestart: () => false,
524
+ stopSignal: "SIGTERM",
525
+ stopTimeoutMs: 2000
526
+ })
527
+ let stopStartedAtMs = 0
528
+
529
+ /**
530
+ * @param {string} phase - The step the deadline was handed to.
531
+ * @param {number | undefined} deadline - Absolute deadline the step received.
532
+ * @returns {void} Records the deadline as an offset from the start of the stop.
533
+ */
534
+ const recordDeadline = (phase, deadline) => {
535
+ if (deadline !== undefined) deadlines.push({msFromStopStart: deadline - stopStartedAtMs, phase})
536
+ }
537
+
538
+ const signalProcessGroup = managed.signalProcessGroup.bind(managed)
539
+ managed.signalProcessGroup = async (signal, pgid, deadline) => {
540
+ signals.push(signal)
541
+ recordDeadline("signal", deadline)
542
+ await signalProcessGroup(signal, pgid, deadline)
543
+ }
544
+
545
+ const waitForProcessGroupExit = managed.waitForProcessGroupExit.bind(managed)
546
+ managed.waitForProcessGroupExit = async (pgid, deadline) => {
547
+ recordDeadline("wait", deadline)
548
+ return await waitForProcessGroupExit(pgid, deadline)
549
+ }
550
+
551
+ try {
552
+ await managed.start()
553
+ // The fixture installs its SIGTERM disposition before it announces readiness. Signalling a
554
+ // child that has not finished booting reaches the default disposition instead, which exits
555
+ // the group and leaves no retirement wait to observe.
556
+ await waitFor(() => fs.existsSync(readyPath))
557
+ stopStartedAtMs = Date.now()
558
+
559
+ await managed.stop({retirementTimeoutMs: 25})
560
+
561
+ // The 2 000 ms graceful window must not outvote the 25 ms retirement budget. Read the other
562
+ // way round the wait runs the whole window before it escalates to SIGKILL, which is the shape
563
+ // that held the TensorBuzz control plane for 4,5 h after the worker moved from "indefinite" to
564
+ // 4,5 h and killed the in-flight builds the consumer said not to interrupt.
565
+ expect(deadlines.map((entry) => entry.phase)).toEqual(["signal", "wait"])
566
+ for (const entry of deadlines) {
567
+ // The deadline starts at stop entry; a millisecond clock tick between this test's marker
568
+ // and the async stop call may shave 1 ms from the observed offset.
569
+ expect(entry.msFromStopStart).toBeGreaterThanOrEqual(0)
570
+ expect(entry.msFromStopStart).toBeLessThan(1000)
571
+ }
572
+ expect(managed.retirementTimedOut).toBe(true)
573
+ expect(signals).toEqual(["SIGTERM"])
574
+ expect(managed.status().pid).toBeDefined()
575
+ } finally {
576
+ const pid = managed.status().pid
577
+ if (pid) process.kill(-pid, "SIGKILL")
578
+ if (managed.exitPromise) await managed.exitPromise
579
+ fs.rmSync(dir, {force: true, recursive: true})
580
+ }
581
+ })
582
+
334
583
  test("a configured stopCommand is used instead of the stop signal", async () => {
335
584
  const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-hooks-"))
336
585
  const logPath = path.join(dir, "stop.log")
@@ -917,4 +1166,14 @@ test("the unlimited constant-delay fast path still applies maxDelayMs", () => {
917
1166
 
918
1167
  expect(queued).toBe(5)
919
1168
  })
1169
+
1170
+ test("the guardian fence releases only a record that is terminal and holds no process group", () => {
1171
+ expect(childRecordIsGone({heldProcessGroup: undefined, pid: undefined, state: "stopped"})).toBe(true)
1172
+ expect(childRecordIsGone({heldProcessGroup: undefined, pid: undefined, state: "failed"})).toBe(true)
1173
+ expect(childRecordIsGone({heldProcessGroup: undefined, pid: 4242, state: "running"})).toBe(false)
1174
+
1175
+ // The shape a drained generation leaves behind: the tracked process record is terminal, but the
1176
+ // group it led still holds live members, so the key is still held by that generation.
1177
+ expect(childRecordIsGone({heldProcessGroup: {liveMemberCount: 1, pgid: 4242}, pid: undefined, state: "stopped"})).toBe(false)
1178
+ })
920
1179
  })