rollbridge 0.1.73 → 0.1.75
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +63 -10
- package/TODO.md +13 -0
- package/changelog.d/20261025-guardian-re-register-dead-generation.md +13 -0
- package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +1 -0
- package/changelog.d/20261104-compose-service-units.md +21 -0
- package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +20 -0
- package/changelog.d/20261106-release-dependency-boot-budgets.md +32 -0
- package/changelog.d/20261109-path-filtered-builds.md +11 -0
- package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +26 -0
- package/changelog.d/20261111-contention-sensitive-daemon-specs.md +27 -0
- package/changelog.d/20261113-bounded-nonblocking-retirement.md +5 -0
- package/changelog.d/20261115-bound-draining-release-groups.md +62 -0
- package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +27 -0
- package/changelog.d/20261115-wedged-generation-escalation.md +44 -0
- package/changelog.d/20261116-bounded-explicit-stop-retirement.md +30 -0
- package/changelog.d/20261116-config-json-canonical-shape.md +11 -0
- package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +29 -0
- package/changelog.d/20261116-guardian-version-handoff.md +37 -0
- package/changelog.d/20261116-reclaim-dead-owner-mutation.md +7 -0
- package/changelog.d/20261117-activate-before-incumbent-retirement.md +1 -0
- package/changelog.d/20261117-activation-path-drain-deadline.md +35 -0
- package/changelog.d/20261117-stop-companion-after-retired-dependency.md +7 -0
- package/changelog.d/20261118-bounded-control-client-close.md +4 -0
- package/changelog.d/20261118-incumbent-retirement-authority-guard.md +1 -0
- package/changelog.d/20261118-owner-retirement-no-guardian-bound.md +19 -0
- package/changelog.d/20261118-shutdown-retirement-bound-restored.md +31 -0
- package/docs/cli.md +128 -3
- package/docs/compose-services.md +222 -0
- package/docs/config.md +177 -6
- package/docs/troubleshooting.md +97 -6
- package/docs/velocious.md +1 -1
- package/docs/workers.md +11 -0
- package/examples/tensorbuzz.com.js +3 -2
- package/package.json +1 -1
- package/src/cli.js +69 -3
- package/src/compose-unit.js +725 -0
- package/src/config.js +346 -8
- package/src/control-client.js +32 -5
- package/src/daemon.js +448 -76
- package/src/doctor.js +74 -0
- package/src/drain-policy.js +252 -0
- package/src/guardian-client.js +92 -9
- package/src/health.js +99 -8
- package/src/managed-process.js +197 -16
- package/src/process-guardian.js +212 -22
- package/src/process-memory.js +62 -2
- package/src/process-order.js +26 -0
- package/src/release-group.js +452 -39
- package/tensorbuzz.yml +20 -1
- package/test/completion.test.js +5 -2
- package/test/compose-config.test.js +160 -0
- package/test/compose-docker.test.js +297 -0
- package/test/compose-unit.test.js +415 -0
- package/test/config-validation.test.js +135 -5
- package/test/control-protocol.test.js +22 -0
- package/test/drain-policy.test.js +328 -0
- package/test/fixtures/deferred-boot-app.js +106 -0
- package/test/fixtures/gated-app.js +50 -0
- package/test/guardian-client.test.js +149 -5
- package/test/health.test.js +74 -1
- package/test/managed-process.test.js +260 -1
- package/test/owner-recovery.test.js +151 -20
- package/test/owner-replacement.test.js +176 -15
- package/test/process-memory.test.js +47 -1
- package/test/release-group.test.js +447 -2
- package/test/rollbridge.test.js +774 -72
- package/test/shutdown-completion.test.js +119 -0
- package/test/shutdown-retirement-bound.test.js +162 -0
- package/test/support/process.js +3 -1
- package/test/tensorbuzz-config.test.js +228 -0
|
@@ -7,19 +7,39 @@ import os from "node:os"
|
|
|
7
7
|
import path from "node:path"
|
|
8
8
|
import {describe, expect, test} from "@velocious/testing"
|
|
9
9
|
import {fileURLToPath} from "node:url"
|
|
10
|
-
import
|
|
10
|
+
import {createRequire} from "node:module"
|
|
11
|
+
import GuardianClient, {usableSocketPath} from "../src/guardian-client.js"
|
|
11
12
|
import {waitForProcessExit} from "./support/process.js"
|
|
12
13
|
|
|
13
14
|
describe("guardian-client", () => {
|
|
14
15
|
|
|
16
|
+
const commonJsRequire = createRequire(import.meta.url)
|
|
17
|
+
const packageVersion = /** @type {{version: string}} */ (commonJsRequire("../package.json")).version
|
|
15
18
|
const legacyGuardianPath = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures", "pre-split3-process-guardian.js")
|
|
16
19
|
const recoveryOwnerPath = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures", "guardian-recovery-owner.js")
|
|
17
20
|
|
|
21
|
+
test("guardian maps an overlong configured socket path to a stable short path", async () => {
|
|
22
|
+
const root = await fs.mkdtemp(path.join(os.tmpdir(), `rollbridge-${"x".repeat(120)}-`))
|
|
23
|
+
const configuredPath = path.join(root, "guardian.sock")
|
|
24
|
+
const client = new GuardianClient({socketPath: configuredPath, token: "long-path-capability"})
|
|
25
|
+
|
|
26
|
+
try {
|
|
27
|
+
expect(Buffer.byteLength(configuredPath)).toBeGreaterThan(107)
|
|
28
|
+
expect(Buffer.byteLength(client.socketPath)).toBeLessThanOrEqual(107)
|
|
29
|
+
expect(client.socketPath).not.toBe(configuredPath)
|
|
30
|
+
await client.launch()
|
|
31
|
+
expect(await client.capabilities()).toEqual({daemonRecovery: 1, generationReactivation: 1, guardianVersion: packageVersion})
|
|
32
|
+
} finally {
|
|
33
|
+
await cleanupGuardian({client, root})
|
|
34
|
+
}
|
|
35
|
+
})
|
|
36
|
+
|
|
18
37
|
test("guardian bootstrap capability is absent from process argv", async () => {
|
|
19
38
|
const fixture = await createGuardian()
|
|
39
|
+
const packageVersion = JSON.parse(await fs.readFile(path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "package.json"), "utf8")).version
|
|
20
40
|
|
|
21
41
|
try {
|
|
22
|
-
expect(await fixture.client.capabilities()).toEqual({daemonRecovery: 1, generationReactivation: 1})
|
|
42
|
+
expect(await fixture.client.capabilities()).toEqual({daemonRecovery: 1, generationReactivation: 1, guardianVersion: packageVersion})
|
|
23
43
|
const commandLine = await fs.readFile(`/proc/${fixture.client.pid}/cmdline`, "utf8")
|
|
24
44
|
const environment = await fs.readFile(`/proc/${fixture.client.pid}/environ`, "utf8")
|
|
25
45
|
const status = await fs.readFile(`/proc/${fixture.client.pid}/status`, "utf8")
|
|
@@ -58,6 +78,100 @@ test("guardian inventory removes only an exact owned provenance", async () => {
|
|
|
58
78
|
}
|
|
59
79
|
})
|
|
60
80
|
|
|
81
|
+
test("a refused owner mutation names the held mutation, its process, its age and its blocker", async () => {
|
|
82
|
+
const fixture = await createGuardian()
|
|
83
|
+
|
|
84
|
+
try {
|
|
85
|
+
const mutationId = await fixture.client.beginOwnerMutation("restart", {ownerPid: process.pid, waitingFor: "the managed process to stop and start again"})
|
|
86
|
+
await expect(fixture.client.beginOwnerMutation("deploy")).rejects.toThrow(/^Another owner mutation is already in progress: owner mutation "restart"; mutation [0-9a-f-]{36}; held by process \d+; since 20\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\.\d\d\dZ; \d+s elapsed; waiting for the managed process to stop and start again; refused deploy$/)
|
|
87
|
+
await fixture.client.reportOwnerMutationWait(mutationId, "an idle forked runner that never exits")
|
|
88
|
+
await expect(fixture.client.beginOwnerMutation("deploy")).rejects.toThrow(/waiting for an idle forked runner that never exits/)
|
|
89
|
+
await fixture.client.endOwnerMutation(mutationId)
|
|
90
|
+
expect(typeof await fixture.client.beginOwnerMutation("deploy")).toBe("string")
|
|
91
|
+
} finally {
|
|
92
|
+
await cleanupGuardian(fixture)
|
|
93
|
+
}
|
|
94
|
+
})
|
|
95
|
+
|
|
96
|
+
test("an owner mutation that outruns its lease is released and the holder reports the expiry", async () => {
|
|
97
|
+
const fixture = await createGuardian()
|
|
98
|
+
|
|
99
|
+
try {
|
|
100
|
+
const wedged = await fixture.client.beginOwnerMutation("restart", {mutationTimeoutMs: 200, ownerPid: process.pid, waitingFor: "a drain that never returns"})
|
|
101
|
+
|
|
102
|
+
await new Promise((resolve) => setTimeout(resolve, 600))
|
|
103
|
+
const successor = await fixture.client.beginOwnerMutation("deploy", {ownerPid: process.pid, waitingFor: "candidate activation"})
|
|
104
|
+
|
|
105
|
+
expect(typeof successor).toBe("string")
|
|
106
|
+
await expect(fixture.client.endOwnerMutation(wedged)).rejects.toThrow(/^Owner mutation restart exceeded its 200 ms lease and was released after \d+ ms; the mutation failed and its lease is gone$/)
|
|
107
|
+
await fixture.client.endOwnerMutation(successor)
|
|
108
|
+
} finally {
|
|
109
|
+
await cleanupGuardian(fixture)
|
|
110
|
+
}
|
|
111
|
+
})
|
|
112
|
+
|
|
113
|
+
test("a dead owner-mutation holder is reclaimed and the next mutation is admitted", async () => {
|
|
114
|
+
const fixture = await createGuardian()
|
|
115
|
+
const holder = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"])
|
|
116
|
+
|
|
117
|
+
try {
|
|
118
|
+
const reclaimed = fixture.client.waitForEvent("owner-mutation-reclaimed")
|
|
119
|
+
await fixture.client.beginOwnerMutation("restart", {ownerPid: holder.pid, waitingFor: "a process that will die"})
|
|
120
|
+
if (!holder.pid) throw new Error("Missing mutation holder PID")
|
|
121
|
+
holder.kill("SIGKILL")
|
|
122
|
+
await once(holder, "exit")
|
|
123
|
+
|
|
124
|
+
const successor = await fixture.client.beginOwnerMutation("deploy", {ownerPid: process.pid, waitingFor: "candidate activation"})
|
|
125
|
+
const event = await reclaimed
|
|
126
|
+
|
|
127
|
+
expect(typeof successor).toBe("string")
|
|
128
|
+
expect(event).toEqual(expect.objectContaining({code: "OWNER_MUTATION_RECLAIMED", event: "owner-mutation-reclaimed", operation: "restart", pid: holder.pid, reason: "owner mutation holder process is no longer alive"}))
|
|
129
|
+
expect((await fixture.client.guardianStatus()).version).toBe(packageVersion)
|
|
130
|
+
await fixture.client.endOwnerMutation(successor)
|
|
131
|
+
} finally {
|
|
132
|
+
if (holder.exitCode === null && holder.signalCode === null) holder.kill("SIGKILL")
|
|
133
|
+
await cleanupGuardian(fixture)
|
|
134
|
+
}
|
|
135
|
+
})
|
|
136
|
+
|
|
137
|
+
test("guardian lets the owner re-register a key whose registered child is gone, and still fences a live one", async () => {
|
|
138
|
+
const fixture = await createGuardian()
|
|
139
|
+
const liveInstance = fixture.client.process("live-generation", definition("live-generation"))
|
|
140
|
+
const deadInstance = fixture.client.process("dead-generation", definition("dead-generation"))
|
|
141
|
+
|
|
142
|
+
try {
|
|
143
|
+
await liveInstance.start()
|
|
144
|
+
await deadInstance.start()
|
|
145
|
+
const deadPid = (await deadInstance.status()).pid
|
|
146
|
+
|
|
147
|
+
if (!deadPid) throw new Error("Missing required fixture: dead-generation pid")
|
|
148
|
+
await deadInstance.stop()
|
|
149
|
+
await waitForProcessExit(deadPid)
|
|
150
|
+
const deadStatus = await deadInstance.status()
|
|
151
|
+
|
|
152
|
+
expect(deadStatus.state).toBe("stopped")
|
|
153
|
+
expect(deadStatus.pid).toBeUndefined()
|
|
154
|
+
|
|
155
|
+
// The wedge this guards against: a later deploy reconstructs the same key from a
|
|
156
|
+
// different release definition. The old child is conclusively gone, so the old
|
|
157
|
+
// provenance must not fence the new registration.
|
|
158
|
+
const nextInstance = fixture.client.process("dead-generation", {...definition("dead-generation"), env: {REPLACED_BY_LATER_DEPLOY: "1"}})
|
|
159
|
+
const reRegistered = nextInstance.status()
|
|
160
|
+
|
|
161
|
+
expect(reRegistered.state).toBe("stopped")
|
|
162
|
+
await nextInstance.start()
|
|
163
|
+
expect((await nextInstance.status()).state).toBe("running")
|
|
164
|
+
expect((await fixture.client.inventory()).some((entry) => entry.key === "dead-generation" && entry.provenance === nextInstance.provenance)).toBe(true)
|
|
165
|
+
|
|
166
|
+
// A live process with a different provenance stays fail-closed.
|
|
167
|
+
const hostileInstance = fixture.client.process("live-generation", {...definition("live-generation"), env: {HOSTILE: "1"}})
|
|
168
|
+
|
|
169
|
+
await expect(hostileInstance.start()).rejects.toThrow(/provenance mismatch/)
|
|
170
|
+
} finally {
|
|
171
|
+
await cleanupGuardian(fixture)
|
|
172
|
+
}
|
|
173
|
+
})
|
|
174
|
+
|
|
61
175
|
test("guardian runs a strict activation lifecycle command for the exact registered process", async () => {
|
|
62
176
|
const fixture = await createGuardian()
|
|
63
177
|
const activationPath = path.join(fixture.root, "activated")
|
|
@@ -842,7 +956,7 @@ test("replacement staging rejects owner state published after prepare", async ()
|
|
|
842
956
|
const prepared = await candidate.prepareOwnerReplacement(authority, nextAuthority)
|
|
843
957
|
|
|
844
958
|
await fixture.client.publishOwnerState({authority, snapshot: {activeReleaseId: "v2"}})
|
|
845
|
-
await expect(candidate.stageOwnerReplacement(prepared.replacementId, {authority: nextAuthority, snapshot: {activeReleaseId: "v1"}})).rejects.toThrow(/owner state changed after prepare/i)
|
|
959
|
+
await expect(candidate.stageOwnerReplacement(prepared.replacementId, {authority: nextAuthority, snapshot: {activeReleaseId: "v1"}})).rejects.toThrow(/owner state changed after prepare \(prepared revision 1, current revision 2\)/i)
|
|
846
960
|
await candidate.abortOwnerReplacement(prepared.replacementId)
|
|
847
961
|
const fresh = await candidate.prepareOwnerReplacement(authority, nextAuthority)
|
|
848
962
|
|
|
@@ -1527,6 +1641,36 @@ test("first upgrade migrates a real pre-split guardian without replacing its own
|
|
|
1527
1641
|
}
|
|
1528
1642
|
})
|
|
1529
1643
|
|
|
1644
|
+
test("a current client re-adopts a stopped record from a legacy guardian", async () => {
|
|
1645
|
+
const fixture = await createLegacyGuardian()
|
|
1646
|
+
const processDefinition = definition("legacy-restore")
|
|
1647
|
+
const legacyProcess = fixture.client.process("release:v1:legacy-restore", processDefinition)
|
|
1648
|
+
let replacement
|
|
1649
|
+
|
|
1650
|
+
try {
|
|
1651
|
+
await legacyProcess.start()
|
|
1652
|
+
const pid = legacyProcess.status().pid
|
|
1653
|
+
|
|
1654
|
+
if (!pid) throw new Error("Missing required legacy process pid")
|
|
1655
|
+
await legacyProcess.stop()
|
|
1656
|
+
await waitForProcessExit(pid)
|
|
1657
|
+
|
|
1658
|
+
replacement = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.client.token})
|
|
1659
|
+
await replacement.connect()
|
|
1660
|
+
const restored = replacement.process("release:v1:legacy-restore", {...processDefinition, env: {RESTORED_BY_CURRENT_CLIENT: "1"}})
|
|
1661
|
+
|
|
1662
|
+
await restored.recover()
|
|
1663
|
+
await restored.start()
|
|
1664
|
+
expect(restored.status().state).toBe("running")
|
|
1665
|
+
} finally {
|
|
1666
|
+
await replacement?.shutdown().catch(() => {})
|
|
1667
|
+
replacement?.disconnect()
|
|
1668
|
+
fixture.client.disconnect()
|
|
1669
|
+
if (fixture.child.exitCode === null && fixture.child.signalCode === null) fixture.child.kill("SIGKILL")
|
|
1670
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
1671
|
+
}
|
|
1672
|
+
})
|
|
1673
|
+
|
|
1530
1674
|
test("split guardian rejects an owner-state update when its nested legacy definition update fails", async () => {
|
|
1531
1675
|
const fixture = await createLegacyGuardian()
|
|
1532
1676
|
const processDefinition = definition("legacy-worker")
|
|
@@ -1678,7 +1822,7 @@ test("a legacy bridge remains discoverable when its candidate disconnects after
|
|
|
1678
1822
|
let replacement
|
|
1679
1823
|
|
|
1680
1824
|
try {
|
|
1681
|
-
const identity = {socketPath: path.join(fixture.root, "guardian-v2.sock"), token: "candidate-guardian-capability"}
|
|
1825
|
+
const identity = {socketPath: usableSocketPath(path.join(fixture.root, "guardian-v2.sock")), token: "candidate-guardian-capability"}
|
|
1682
1826
|
|
|
1683
1827
|
upgraded = await fixture.client.upgradeLegacyGuardian({ownerState, ...identity})
|
|
1684
1828
|
const prepared = await upgraded.prepareOwnerReplacement(authority, nextAuthority)
|
|
@@ -1725,7 +1869,7 @@ async function createGuardian() {
|
|
|
1725
1869
|
async function createLegacyGuardian() {
|
|
1726
1870
|
const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-legacy-guardian-"))
|
|
1727
1871
|
const token = "legacy-guardian-capability"
|
|
1728
|
-
const socketPath = path.join(root, "guardian.sock")
|
|
1872
|
+
const socketPath = usableSocketPath(path.join(root, "guardian.sock"))
|
|
1729
1873
|
const child = spawn(process.execPath, [legacyGuardianPath, socketPath], {detached: true, stdio: ["ignore", "ignore", "ignore", "ipc"]})
|
|
1730
1874
|
|
|
1731
1875
|
await new Promise((resolve, reject) => {
|
package/test/health.test.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
3
|
import http from "node:http"
|
|
4
|
+
import net from "node:net"
|
|
4
5
|
import {describe, expect, test} from "@velocious/testing"
|
|
5
|
-
import {waitForHealth} from "../src/health.js"
|
|
6
|
+
import {waitForHealth, waitForListeningPort} from "../src/health.js"
|
|
6
7
|
|
|
7
8
|
describe("health", () => {
|
|
8
9
|
|
|
@@ -31,6 +32,18 @@ async function startHealthServer() {
|
|
|
31
32
|
}
|
|
32
33
|
}
|
|
33
34
|
|
|
35
|
+
/**
|
|
36
|
+
* @param {net.Server} server - Server to bind.
|
|
37
|
+
* @returns {Promise<number>} Ephemeral port it bound.
|
|
38
|
+
*/
|
|
39
|
+
async function bindPort(server) {
|
|
40
|
+
await new Promise((resolve) => server.listen(0, "127.0.0.1", () => resolve(undefined)))
|
|
41
|
+
|
|
42
|
+
const address = server.address()
|
|
43
|
+
|
|
44
|
+
return address && typeof address === "object" ? address.port : 0
|
|
45
|
+
}
|
|
46
|
+
|
|
34
47
|
test("waitForHealth delays the first probe by startDelayMs", async () => {
|
|
35
48
|
const server = await startHealthServer()
|
|
36
49
|
|
|
@@ -62,4 +75,64 @@ test("waitForHealth probes immediately when startDelayMs is 0", async () => {
|
|
|
62
75
|
await server.close()
|
|
63
76
|
}
|
|
64
77
|
})
|
|
78
|
+
|
|
79
|
+
test("waitForHealth names the deadline and the time it spent when the app never becomes healthy", async () => {
|
|
80
|
+
const server = http.createServer((_request, response) => {
|
|
81
|
+
response.writeHead(500)
|
|
82
|
+
response.end("bad")
|
|
83
|
+
})
|
|
84
|
+
|
|
85
|
+
try {
|
|
86
|
+
const port = await bindPort(server)
|
|
87
|
+
|
|
88
|
+
/** @type {import("../src/health.js").HealthCheckError} */
|
|
89
|
+
const failure = /** @type {import("../src/health.js").HealthCheckError} */ (await waitForHealth({
|
|
90
|
+
health: {intervalMs: 25, path: "/ping", startDelayMs: 0, timeoutMs: 300},
|
|
91
|
+
host: "127.0.0.1",
|
|
92
|
+
port,
|
|
93
|
+
processId: "web"
|
|
94
|
+
}).catch((error) => error instanceof Error ? error : new Error(String(error))))
|
|
95
|
+
|
|
96
|
+
expect(String(failure.message)).toMatch(/process web: \d+ probes, \d+ms elapsed of the 300ms health deadline/)
|
|
97
|
+
expect(failure.healthCheck?.deadlineMs).toBe(300)
|
|
98
|
+
expect({value: Boolean(failure.healthCheck && failure.healthCheck.elapsedMs >= 300), context: `expected the health gate to have spent its whole budget, was ${failure.healthCheck?.elapsedMs}ms`}).toMatchObject({value: true})
|
|
99
|
+
expect({value: Boolean(failure.healthCheck && failure.healthCheck.attempts >= 1), context: `expected at least one probe, was ${failure.healthCheck?.attempts}`}).toMatchObject({value: true})
|
|
100
|
+
expect(failure.healthCheck?.processId).toBe("web")
|
|
101
|
+
} finally {
|
|
102
|
+
await server.close()
|
|
103
|
+
}
|
|
104
|
+
})
|
|
105
|
+
|
|
106
|
+
test("waitForListeningPort waits for a dependency port on a budget of its own", async () => {
|
|
107
|
+
const server = net.createServer()
|
|
108
|
+
const port = await bindPort(server)
|
|
109
|
+
|
|
110
|
+
await new Promise((resolve) => server.close(() => resolve(undefined)))
|
|
111
|
+
|
|
112
|
+
const waiting = waitForListeningPort({host: "127.0.0.1", intervalMs: 25, port, timeoutMs: 2000})
|
|
113
|
+
|
|
114
|
+
await new Promise((resolve) => setTimeout(resolve, 300))
|
|
115
|
+
await new Promise((resolve) => server.listen(port, "127.0.0.1", () => resolve(undefined)))
|
|
116
|
+
|
|
117
|
+
try {
|
|
118
|
+
const result = await waiting
|
|
119
|
+
|
|
120
|
+
expect({value: Boolean(result.attempts >= 2), context: `expected more than one connection attempt, was ${result.attempts}`}).toMatchObject({value: true})
|
|
121
|
+
expect({value: Boolean(result.elapsedMs >= 250), context: `expected the dependency wait to take ~300ms, was ${result.elapsedMs}ms`}).toMatchObject({value: true})
|
|
122
|
+
} finally {
|
|
123
|
+
await new Promise((resolve) => server.close(() => resolve(undefined)))
|
|
124
|
+
}
|
|
125
|
+
})
|
|
126
|
+
|
|
127
|
+
test("waitForListeningPort names the budget it ran out on when the dependency never binds", async () => {
|
|
128
|
+
const server = net.createServer()
|
|
129
|
+
const port = await bindPort(server)
|
|
130
|
+
|
|
131
|
+
await new Promise((resolve) => server.close(() => resolve(undefined)))
|
|
132
|
+
|
|
133
|
+
/** @type {Error} */
|
|
134
|
+
const failure = /** @type {Error} */ (await waitForListeningPort({host: "127.0.0.1", intervalMs: 25, port, timeoutMs: 200}).catch((error) => error instanceof Error ? error : new Error(String(error))))
|
|
135
|
+
|
|
136
|
+
expect(String(failure.message)).toMatch(/did not accept a connection within 200ms/)
|
|
137
|
+
})
|
|
65
138
|
})
|
|
@@ -6,7 +6,7 @@ import os from "node:os"
|
|
|
6
6
|
import path from "node:path"
|
|
7
7
|
import {describe, expect, test} from "@velocious/testing"
|
|
8
8
|
import {fileURLToPath} from "node:url"
|
|
9
|
-
import ManagedProcess from "../src/managed-process.js"
|
|
9
|
+
import ManagedProcess, {childRecordIsGone} from "../src/managed-process.js"
|
|
10
10
|
import {processStartTime} from "../src/process-memory.js"
|
|
11
11
|
|
|
12
12
|
describe("managed-process", () => {
|
|
@@ -331,6 +331,255 @@ test("runs quiet and drain lifecycle hooks before stopping", async () => {
|
|
|
331
331
|
}
|
|
332
332
|
})
|
|
333
333
|
|
|
334
|
+
test("retirement timeout leaves an indefinite worker alive without SIGKILL", async () => {
|
|
335
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-retirement-wedge-"))
|
|
336
|
+
const readyPath = path.join(dir, "ready")
|
|
337
|
+
/** @type {string[]} */
|
|
338
|
+
const signals = []
|
|
339
|
+
const managed = new ManagedProcess({
|
|
340
|
+
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`process.on('SIGTERM', () => {}); require("node:fs").writeFileSync(${JSON.stringify(readyPath)}, "ready"); setInterval(() => {}, 1000)`)}`,
|
|
341
|
+
cwd: undefined,
|
|
342
|
+
env: {},
|
|
343
|
+
id: "worker",
|
|
344
|
+
lifecycle: {drainTimeoutMs: 0},
|
|
345
|
+
logger: () => {},
|
|
346
|
+
outputLines: 50,
|
|
347
|
+
restartDelayMs: 10,
|
|
348
|
+
shouldRestart: () => false,
|
|
349
|
+
stopSignal: "SIGTERM",
|
|
350
|
+
stopTimeoutMs: "indefinite"
|
|
351
|
+
})
|
|
352
|
+
const signalProcessGroup = managed.signalProcessGroup.bind(managed)
|
|
353
|
+
managed.signalProcessGroup = async (signal, pgid, deadline) => {
|
|
354
|
+
signals.push(signal)
|
|
355
|
+
await signalProcessGroup(signal, pgid, deadline)
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
try {
|
|
359
|
+
await managed.start()
|
|
360
|
+
// The fixture installs its SIGTERM disposition before it announces readiness. Signalling a
|
|
361
|
+
// child that has not finished booting reaches the default disposition instead, which exits
|
|
362
|
+
// the group and leaves no wedge to observe.
|
|
363
|
+
await waitFor(() => fs.existsSync(readyPath))
|
|
364
|
+
await managed.stop({retirementTimeoutMs: 25})
|
|
365
|
+
|
|
366
|
+
expect(managed.retirementTimedOut).toBe(true)
|
|
367
|
+
expect(signals).toEqual(["SIGTERM"])
|
|
368
|
+
expect(managed.status().pid).toBeDefined()
|
|
369
|
+
} finally {
|
|
370
|
+
const pid = managed.status().pid
|
|
371
|
+
if (pid) process.kill(-pid, "SIGKILL")
|
|
372
|
+
if (managed.exitPromise) await managed.exitPromise
|
|
373
|
+
fs.rmSync(dir, {force: true, recursive: true})
|
|
374
|
+
}
|
|
375
|
+
})
|
|
376
|
+
|
|
377
|
+
test("escalateRetirement terminates the wedged worker the bounded retirement left alive", async () => {
|
|
378
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-escalation-wedge-"))
|
|
379
|
+
const readyPath = path.join(dir, "ready")
|
|
380
|
+
/** @type {string[]} */
|
|
381
|
+
const signals = []
|
|
382
|
+
/** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
|
|
383
|
+
const logs = []
|
|
384
|
+
const managed = new ManagedProcess({
|
|
385
|
+
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`process.on('SIGTERM', () => {}); require("node:fs").writeFileSync(${JSON.stringify(readyPath)}, "ready"); setInterval(() => {}, 1000)`)}`,
|
|
386
|
+
cwd: undefined,
|
|
387
|
+
env: {},
|
|
388
|
+
id: "worker",
|
|
389
|
+
lifecycle: {drainTimeoutMs: 0},
|
|
390
|
+
logger: (message, data) => logs.push({data, message}),
|
|
391
|
+
outputLines: 50,
|
|
392
|
+
restartDelayMs: 10,
|
|
393
|
+
shouldRestart: () => false,
|
|
394
|
+
stopSignal: "SIGTERM",
|
|
395
|
+
stopTimeoutMs: "indefinite"
|
|
396
|
+
})
|
|
397
|
+
const signalProcessGroup = managed.signalProcessGroup.bind(managed)
|
|
398
|
+
|
|
399
|
+
managed.signalProcessGroup = async (signal, pgid, deadline) => {
|
|
400
|
+
signals.push(signal)
|
|
401
|
+
await signalProcessGroup(signal, pgid, deadline)
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
try {
|
|
405
|
+
await managed.start()
|
|
406
|
+
await waitFor(() => fs.existsSync(readyPath))
|
|
407
|
+
await managed.stop({retirementTimeoutMs: 25})
|
|
408
|
+
const wedgedPid = managed.status().pid
|
|
409
|
+
|
|
410
|
+
expect(managed.retirementTimedOut).toBe(true)
|
|
411
|
+
expect(signals).toEqual(["SIGTERM"])
|
|
412
|
+
expect(wedgedPid).toBeDefined()
|
|
413
|
+
|
|
414
|
+
expect(await managed.escalateRetirement({reason: "spec: proven wedge"})).toBe(true)
|
|
415
|
+
expect(signals).toEqual(["SIGTERM", "SIGKILL"])
|
|
416
|
+
expect(logs.some((entry) => entry.message === "process retirement escalated to SIGKILL" && entry.data?.reason === "spec: proven wedge")).toBe(true)
|
|
417
|
+
expect(await managed.waitForProcessGroupExit(/** @type {number} */ (wedgedPid), Date.now() + 5000)).toBe(true)
|
|
418
|
+
if (managed.exitPromise) await managed.exitPromise
|
|
419
|
+
expect(managed.status().pid).toBeUndefined()
|
|
420
|
+
expect(managed.status().state).toBe("stopped")
|
|
421
|
+
} finally {
|
|
422
|
+
const pid = managed.status().pid
|
|
423
|
+
if (pid) process.kill(-pid, "SIGKILL")
|
|
424
|
+
if (managed.exitPromise) await managed.exitPromise
|
|
425
|
+
fs.rmSync(dir, {force: true, recursive: true})
|
|
426
|
+
}
|
|
427
|
+
})
|
|
428
|
+
|
|
429
|
+
test("escalateRetirement refuses a process that has not expired its retirement budget", async () => {
|
|
430
|
+
/** @type {string[]} */
|
|
431
|
+
const signals = []
|
|
432
|
+
/** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
|
|
433
|
+
const logs = []
|
|
434
|
+
const managed = new ManagedProcess({
|
|
435
|
+
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify("setInterval(() => {}, 1000)")}`,
|
|
436
|
+
cwd: undefined,
|
|
437
|
+
env: {},
|
|
438
|
+
id: "worker",
|
|
439
|
+
lifecycle: {drainTimeoutMs: 0},
|
|
440
|
+
logger: (message, data) => logs.push({data, message}),
|
|
441
|
+
outputLines: 50,
|
|
442
|
+
restartDelayMs: 10,
|
|
443
|
+
shouldRestart: () => false,
|
|
444
|
+
stopSignal: "SIGTERM",
|
|
445
|
+
stopTimeoutMs: "indefinite"
|
|
446
|
+
})
|
|
447
|
+
const signalProcessGroup = managed.signalProcessGroup.bind(managed)
|
|
448
|
+
|
|
449
|
+
managed.signalProcessGroup = async (signal, pgid, deadline) => {
|
|
450
|
+
signals.push(signal)
|
|
451
|
+
await signalProcessGroup(signal, pgid, deadline)
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
try {
|
|
455
|
+
await managed.start()
|
|
456
|
+
await managed.quiesce()
|
|
457
|
+
|
|
458
|
+
expect(managed.retirementTimedOut).toBe(false)
|
|
459
|
+
expect(await managed.escalateRetirement({reason: "spec"})).toBe(false)
|
|
460
|
+
expect(signals).toEqual([])
|
|
461
|
+
expect(logs.some((entry) => entry.message === "retirement escalation refused; process is not past its retirement budget")).toBe(true)
|
|
462
|
+
expect(managed.status().pid).toBeDefined()
|
|
463
|
+
} finally {
|
|
464
|
+
const pid = managed.status().pid
|
|
465
|
+
if (pid) process.kill(-pid, "SIGKILL")
|
|
466
|
+
if (managed.exitPromise) await managed.exitPromise
|
|
467
|
+
}
|
|
468
|
+
})
|
|
469
|
+
|
|
470
|
+
test("escalateRetirement is inert for a process that drained normally", async () => {
|
|
471
|
+
/** @type {string[]} */
|
|
472
|
+
const signals = []
|
|
473
|
+
const managed = new ManagedProcess({
|
|
474
|
+
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify("setInterval(() => {}, 1000)")}`,
|
|
475
|
+
cwd: undefined,
|
|
476
|
+
env: {},
|
|
477
|
+
id: "worker",
|
|
478
|
+
lifecycle: {drainTimeoutMs: 0},
|
|
479
|
+
logger: () => {},
|
|
480
|
+
outputLines: 50,
|
|
481
|
+
restartDelayMs: 10,
|
|
482
|
+
shouldRestart: () => false,
|
|
483
|
+
stopSignal: "SIGTERM",
|
|
484
|
+
stopTimeoutMs: 2000
|
|
485
|
+
})
|
|
486
|
+
const signalProcessGroup = managed.signalProcessGroup.bind(managed)
|
|
487
|
+
|
|
488
|
+
managed.signalProcessGroup = async (signal, pgid, deadline) => {
|
|
489
|
+
signals.push(signal)
|
|
490
|
+
await signalProcessGroup(signal, pgid, deadline)
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
try {
|
|
494
|
+
await managed.start()
|
|
495
|
+
await managed.stop()
|
|
496
|
+
|
|
497
|
+
expect(managed.status().state).toBe("stopped")
|
|
498
|
+
expect(await managed.escalateRetirement({reason: "spec"})).toBe(false)
|
|
499
|
+
expect(signals).toEqual(["SIGTERM"])
|
|
500
|
+
} finally {
|
|
501
|
+
const pid = managed.status().pid
|
|
502
|
+
if (pid) process.kill(-pid, "SIGKILL")
|
|
503
|
+
if (managed.exitPromise) await managed.exitPromise
|
|
504
|
+
}
|
|
505
|
+
})
|
|
506
|
+
|
|
507
|
+
test("a retirement budget caps a finite graceful window instead of being outvoted by it", async () => {
|
|
508
|
+
/** @type {string[]} */
|
|
509
|
+
const signals = []
|
|
510
|
+
/** @type {Array<{msFromStopStart: number, phase: string}>} */
|
|
511
|
+
const deadlines = []
|
|
512
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-retirement-bound-"))
|
|
513
|
+
const readyPath = path.join(dir, "ready")
|
|
514
|
+
const managed = new ManagedProcess({
|
|
515
|
+
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`process.on('SIGTERM', () => {}); require("node:fs").writeFileSync(${JSON.stringify(readyPath)}, "ready"); setInterval(() => {}, 1000)`)}`,
|
|
516
|
+
cwd: undefined,
|
|
517
|
+
env: {},
|
|
518
|
+
id: "worker",
|
|
519
|
+
lifecycle: {drainTimeoutMs: 0},
|
|
520
|
+
logger: () => {},
|
|
521
|
+
outputLines: 50,
|
|
522
|
+
restartDelayMs: 10,
|
|
523
|
+
shouldRestart: () => false,
|
|
524
|
+
stopSignal: "SIGTERM",
|
|
525
|
+
stopTimeoutMs: 2000
|
|
526
|
+
})
|
|
527
|
+
let stopStartedAtMs = 0
|
|
528
|
+
|
|
529
|
+
/**
|
|
530
|
+
* @param {string} phase - The step the deadline was handed to.
|
|
531
|
+
* @param {number | undefined} deadline - Absolute deadline the step received.
|
|
532
|
+
* @returns {void} Records the deadline as an offset from the start of the stop.
|
|
533
|
+
*/
|
|
534
|
+
const recordDeadline = (phase, deadline) => {
|
|
535
|
+
if (deadline !== undefined) deadlines.push({msFromStopStart: deadline - stopStartedAtMs, phase})
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
const signalProcessGroup = managed.signalProcessGroup.bind(managed)
|
|
539
|
+
managed.signalProcessGroup = async (signal, pgid, deadline) => {
|
|
540
|
+
signals.push(signal)
|
|
541
|
+
recordDeadline("signal", deadline)
|
|
542
|
+
await signalProcessGroup(signal, pgid, deadline)
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
const waitForProcessGroupExit = managed.waitForProcessGroupExit.bind(managed)
|
|
546
|
+
managed.waitForProcessGroupExit = async (pgid, deadline) => {
|
|
547
|
+
recordDeadline("wait", deadline)
|
|
548
|
+
return await waitForProcessGroupExit(pgid, deadline)
|
|
549
|
+
}
|
|
550
|
+
|
|
551
|
+
try {
|
|
552
|
+
await managed.start()
|
|
553
|
+
// The fixture installs its SIGTERM disposition before it announces readiness. Signalling a
|
|
554
|
+
// child that has not finished booting reaches the default disposition instead, which exits
|
|
555
|
+
// the group and leaves no retirement wait to observe.
|
|
556
|
+
await waitFor(() => fs.existsSync(readyPath))
|
|
557
|
+
stopStartedAtMs = Date.now()
|
|
558
|
+
|
|
559
|
+
await managed.stop({retirementTimeoutMs: 25})
|
|
560
|
+
|
|
561
|
+
// The 2 000 ms graceful window must not outvote the 25 ms retirement budget. Read the other
|
|
562
|
+
// way round the wait runs the whole window before it escalates to SIGKILL, which is the shape
|
|
563
|
+
// that held the TensorBuzz control plane for 4,5 h after the worker moved from "indefinite" to
|
|
564
|
+
// 4,5 h and killed the in-flight builds the consumer said not to interrupt.
|
|
565
|
+
expect(deadlines.map((entry) => entry.phase)).toEqual(["signal", "wait"])
|
|
566
|
+
for (const entry of deadlines) {
|
|
567
|
+
// The deadline starts at stop entry; a millisecond clock tick between this test's marker
|
|
568
|
+
// and the async stop call may shave 1 ms from the observed offset.
|
|
569
|
+
expect(entry.msFromStopStart).toBeGreaterThanOrEqual(0)
|
|
570
|
+
expect(entry.msFromStopStart).toBeLessThan(1000)
|
|
571
|
+
}
|
|
572
|
+
expect(managed.retirementTimedOut).toBe(true)
|
|
573
|
+
expect(signals).toEqual(["SIGTERM"])
|
|
574
|
+
expect(managed.status().pid).toBeDefined()
|
|
575
|
+
} finally {
|
|
576
|
+
const pid = managed.status().pid
|
|
577
|
+
if (pid) process.kill(-pid, "SIGKILL")
|
|
578
|
+
if (managed.exitPromise) await managed.exitPromise
|
|
579
|
+
fs.rmSync(dir, {force: true, recursive: true})
|
|
580
|
+
}
|
|
581
|
+
})
|
|
582
|
+
|
|
334
583
|
test("a configured stopCommand is used instead of the stop signal", async () => {
|
|
335
584
|
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-hooks-"))
|
|
336
585
|
const logPath = path.join(dir, "stop.log")
|
|
@@ -917,4 +1166,14 @@ test("the unlimited constant-delay fast path still applies maxDelayMs", () => {
|
|
|
917
1166
|
|
|
918
1167
|
expect(queued).toBe(5)
|
|
919
1168
|
})
|
|
1169
|
+
|
|
1170
|
+
test("the guardian fence releases only a record that is terminal and holds no process group", () => {
|
|
1171
|
+
expect(childRecordIsGone({heldProcessGroup: undefined, pid: undefined, state: "stopped"})).toBe(true)
|
|
1172
|
+
expect(childRecordIsGone({heldProcessGroup: undefined, pid: undefined, state: "failed"})).toBe(true)
|
|
1173
|
+
expect(childRecordIsGone({heldProcessGroup: undefined, pid: 4242, state: "running"})).toBe(false)
|
|
1174
|
+
|
|
1175
|
+
// The shape a drained generation leaves behind: the tracked process record is terminal, but the
|
|
1176
|
+
// group it led still holds live members, so the key is still held by that generation.
|
|
1177
|
+
expect(childRecordIsGone({heldProcessGroup: {liveMemberCount: 1, pgid: 4242}, pid: undefined, state: "stopped"})).toBe(false)
|
|
1178
|
+
})
|
|
920
1179
|
})
|