rollbridge 0.1.72 → 0.1.74

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/README.md +63 -10
  2. package/TODO.md +13 -0
  3. package/changelog.d/20261025-guardian-re-register-dead-generation.md +13 -0
  4. package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +1 -0
  5. package/changelog.d/20261104-compose-service-units.md +21 -0
  6. package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +20 -0
  7. package/changelog.d/20261106-release-dependency-boot-budgets.md +32 -0
  8. package/changelog.d/20261109-path-filtered-builds.md +11 -0
  9. package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +26 -0
  10. package/changelog.d/20261111-contention-sensitive-daemon-specs.md +27 -0
  11. package/changelog.d/20261113-bounded-nonblocking-retirement.md +5 -0
  12. package/changelog.d/20261115-bound-draining-release-groups.md +62 -0
  13. package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +27 -0
  14. package/changelog.d/20261115-wedged-generation-escalation.md +44 -0
  15. package/changelog.d/20261116-bounded-explicit-stop-retirement.md +30 -0
  16. package/changelog.d/20261116-config-json-canonical-shape.md +11 -0
  17. package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +29 -0
  18. package/changelog.d/20261116-guardian-version-handoff.md +37 -0
  19. package/changelog.d/20261116-reclaim-dead-owner-mutation.md +7 -0
  20. package/changelog.d/20261117-activate-before-incumbent-retirement.md +1 -0
  21. package/changelog.d/20261117-activation-path-drain-deadline.md +35 -0
  22. package/changelog.d/20261117-stop-companion-after-retired-dependency.md +7 -0
  23. package/changelog.d/20261118-incumbent-retirement-authority-guard.md +1 -0
  24. package/changelog.d/20261118-owner-retirement-no-guardian-bound.md +19 -0
  25. package/changelog.d/20261118-shutdown-retirement-bound-restored.md +31 -0
  26. package/docs/cli.md +128 -3
  27. package/docs/compose-services.md +222 -0
  28. package/docs/config.md +177 -6
  29. package/docs/troubleshooting.md +97 -6
  30. package/docs/velocious.md +1 -1
  31. package/docs/workers.md +11 -0
  32. package/examples/tensorbuzz.com.js +3 -2
  33. package/package.json +1 -1
  34. package/src/cli.js +68 -2
  35. package/src/compose-unit.js +725 -0
  36. package/src/config.js +346 -8
  37. package/src/daemon.js +410 -73
  38. package/src/doctor.js +74 -0
  39. package/src/drain-policy.js +252 -0
  40. package/src/guardian-client.js +67 -8
  41. package/src/health.js +99 -8
  42. package/src/managed-process.js +197 -16
  43. package/src/process-guardian.js +212 -22
  44. package/src/process-memory.js +62 -2
  45. package/src/process-order.js +26 -0
  46. package/src/release-group.js +435 -37
  47. package/tensorbuzz.yml +20 -1
  48. package/test/completion.test.js +5 -2
  49. package/test/compose-config.test.js +160 -0
  50. package/test/compose-docker.test.js +297 -0
  51. package/test/compose-unit.test.js +415 -0
  52. package/test/config-validation.test.js +135 -5
  53. package/test/drain-policy.test.js +328 -0
  54. package/test/fixtures/deferred-boot-app.js +106 -0
  55. package/test/fixtures/gated-app.js +50 -0
  56. package/test/guardian-client.test.js +119 -5
  57. package/test/health.test.js +74 -1
  58. package/test/managed-process.test.js +260 -1
  59. package/test/owner-recovery.test.js +125 -21
  60. package/test/owner-replacement.test.js +175 -14
  61. package/test/process-memory.test.js +47 -1
  62. package/test/release-group.test.js +408 -2
  63. package/test/rollbridge.test.js +774 -72
  64. package/test/shutdown-completion.test.js +119 -0
  65. package/test/shutdown-retirement-bound.test.js +162 -0
  66. package/test/support/process.js +3 -1
  67. package/test/tensorbuzz-config.test.js +228 -0
@@ -0,0 +1,328 @@
1
+ // @ts-check
2
+
3
+ import {describe, expect, test} from "@velocious/testing"
4
+ import {admitGenerationRelease, classifyDrainProgress, decideWedgedEscalation} from "../src/drain-policy.js"
5
+
6
+ describe("drain-policy", () => {
7
+
8
+ const WINDOW_MS = 20000
9
+
10
+ /**
11
+ * @param {object} [options] - Observation shape.
12
+ * @param {number} [options.atMs] - Observation time.
13
+ * @param {number | undefined} [options.cpuMs] - Cumulative CPU of the group.
14
+ * @param {number | undefined} [options.heldProcessGroupPgid] - Group still alive after its tracked process exited.
15
+ * @param {number} [options.liveChildCount] - Runner children other than the group leader.
16
+ * @param {number | undefined} [options.pid] - Live pid, or undefined when the process is gone.
17
+ * @param {boolean} [options.retirementTimedOut] - Whether the supervisor already gave up on it.
18
+ * @returns {import("../src/drain-policy.js").DrainLivenessSample} One liveness observation.
19
+ */
20
+ function sample(options = {}) {
21
+ return {
22
+ atMs: options.atMs ?? 1000000,
23
+ processes: [{
24
+ cpuMs: "cpuMs" in options ? options.cpuMs : 5000,
25
+ heldProcessGroupPgid: options.heldProcessGroupPgid,
26
+ id: "worker",
27
+ liveChildCount: options.liveChildCount ?? 1,
28
+ pid: "pid" in options ? options.pid : 4242,
29
+ retirementTimedOut: options.retirementTimedOut ?? true
30
+ }]
31
+ }
32
+ }
33
+
34
+ /**
35
+ * @param {object} [options] - Classification inputs.
36
+ * @param {number} [options.connectionCount] - Live connections owned by the generation.
37
+ * @param {import("../src/drain-policy.js").DrainLivenessSample} [options.current] - Newest observation.
38
+ * @param {number} [options.drainBudgetMs] - Configured drain budget.
39
+ * @param {number} [options.drainElapsedMs] - Time since the drain started.
40
+ * @param {import("../src/drain-policy.js").DrainLivenessSample | undefined} [options.previous] - Older observation.
41
+ * @returns {import("../src/drain-policy.js").DrainClassification} Classification.
42
+ */
43
+ function classify(options = {}) {
44
+ return classifyDrainProgress({
45
+ connectionCount: options.connectionCount ?? 0,
46
+ currentSample: options.current ?? sample(),
47
+ drainBudgetMs: options.drainBudgetMs ?? 0,
48
+ drainElapsedMs: options.drainElapsedMs ?? 0,
49
+ minSampleWindowMs: WINDOW_MS,
50
+ previousSample: options.previous
51
+ })
52
+ }
53
+
54
+ /**
55
+ * @param {object} [options] - Decision inputs.
56
+ * @param {number} [options.connectionCount] - Live connections owned by the generation.
57
+ * @param {import("../src/drain-policy.js").DrainLivenessSample} [options.current] - Newest observation.
58
+ * @param {import("../src/drain-policy.js").DrainLivenessSample | undefined} [options.previous] - Older observation.
59
+ * @returns {import("../src/drain-policy.js").EscalationDecision} Escalation decision for the same pair the classifier saw.
60
+ */
61
+ function escalate(options = {}) {
62
+ const currentSample = options.current ?? sample()
63
+
64
+ return decideWedgedEscalation({
65
+ classification: classify({connectionCount: options.connectionCount, current: currentSample, previous: options.previous}),
66
+ currentSample,
67
+ minSampleWindowMs: WINDOW_MS
68
+ })
69
+ }
70
+
71
+ test("a generation that still owns connections is draining, never wedged", () => {
72
+ const result = classify({connectionCount: 3, current: sample({liveChildCount: 0})})
73
+
74
+ expect(result.status).toBe("progressing")
75
+ expect(result.reason).toBe("3 live connections still owned by the generation")
76
+ })
77
+
78
+ test("a live connection past the configured drain budget is reported past the deadline, not progressing", () => {
79
+ const result = classify({connectionCount: 1, current: sample({liveChildCount: 0}), drainBudgetMs: 60000, drainElapsedMs: 61000})
80
+
81
+ expect(result.status).toBe("timedOut")
82
+ expect(result.reason).toBe("drain budget 60000ms exceeded by 1000ms with 1 live connection still owned by the generation")
83
+ })
84
+
85
+ test("a live connection inside the configured drain budget is still a legitimate drain", () => {
86
+ const result = classify({connectionCount: 1, current: sample({liveChildCount: 0}), drainBudgetMs: 60000, drainElapsedMs: 30000})
87
+
88
+ expect(result.status).toBe("progressing")
89
+ expect(result.reason).toBe("1 live connection still owned by the generation")
90
+ })
91
+
92
+ test("the drain deadline names the deadline and never escalates", () => {
93
+ const currentSample = sample({liveChildCount: 0})
94
+ const classification = classify({connectionCount: 1, current: currentSample, drainBudgetMs: 60000, drainElapsedMs: 61000})
95
+ const decision = decideWedgedEscalation({classification, currentSample, minSampleWindowMs: WINDOW_MS})
96
+
97
+ expect(decision.decision).toBe("report")
98
+ expect(decision.reason).toBe("no escalation: drain is timedOut (drain budget 60000ms exceeded by 1000ms with 1 live connection still owned by the generation)")
99
+ expect(decision.targets).toEqual([])
100
+ })
101
+
102
+ test("a process still inside its retirement budget is draining", () => {
103
+ const result = classify({current: sample({liveChildCount: 0, retirementTimedOut: false})})
104
+
105
+ expect(result.status).toBe("progressing")
106
+ expect(result.reason).toBe("process worker is still inside its retirement budget")
107
+ })
108
+
109
+ test("a live process past its retirement budget with no runner child left is wedged", () => {
110
+ const result = classify({current: sample({liveChildCount: 0})})
111
+
112
+ expect(result.status).toBe("wedged")
113
+ expect(result.reason).toBe("1 live process past its retirement budget with no runner child remaining")
114
+ })
115
+
116
+ test("flat CPU across a sufficient window is wedged, and the window is reported", () => {
117
+ const previous = sample({atMs: 1000000, cpuMs: 5000})
118
+ const current = sample({atMs: 1025000, cpuMs: 5000, liveChildCount: 3})
119
+ const result = classify({current, previous})
120
+
121
+ expect(result.status).toBe("wedged")
122
+ expect(result.sampleWindowMs).toBe(25000)
123
+ expect(result.cpuDeltaMs).toBe(0)
124
+ expect(result.reason).toBe("1 live process with 3 runner children, no live connection, every retirement budget expired, CPU flat across 25000ms")
125
+ })
126
+
127
+ test("CPU that advanced across the window is a progressing drain", () => {
128
+ const previous = sample({atMs: 1000000, cpuMs: 5000})
129
+ const current = sample({atMs: 1025000, cpuMs: 9000})
130
+ const result = classify({current, previous})
131
+
132
+ expect(result.status).toBe("progressing")
133
+ expect(result.cpuDeltaMs).toBe(4000)
134
+ expect(result.reason).toBe("CPU advanced 4000ms across 25000ms")
135
+ })
136
+
137
+ test("CPU leaving the group with an exited member is a progressing drain", () => {
138
+ const previous = sample({atMs: 1000000, cpuMs: 9000})
139
+ const current = sample({atMs: 1025000, cpuMs: 5000})
140
+ const result = classify({current, previous})
141
+
142
+ expect(result.status).toBe("progressing")
143
+ expect(result.cpuDeltaMs).toBe(-4000)
144
+ expect(result.reason).toBe("the group changed (1 live process, -4000ms of consumed CPU left the group) across 25000ms")
145
+ })
146
+
147
+ test("the first observation cannot claim flatness", () => {
148
+ const result = classify({current: sample()})
149
+
150
+ expect(result.status).toBe("unproven")
151
+ expect(result.sampleWindowMs).toBe(0)
152
+ expect(result.reason).toBe("first liveness observation; a 20000ms window is required before CPU flatness can be claimed")
153
+ })
154
+
155
+ test("a window shorter than the minimum cannot claim flatness", () => {
156
+ const previous = sample({atMs: 1000000})
157
+ const current = sample({atMs: 1010000})
158
+ const result = classify({current, previous})
159
+
160
+ expect(result.status).toBe("unproven")
161
+ expect(result.sampleWindowMs).toBe(10000)
162
+ expect(result.reason).toBe("liveness window 10000ms is below the 20000ms minimum")
163
+ })
164
+
165
+ test("an unmeasurable CPU counter is unproven, never wedged", () => {
166
+ const previous = sample({atMs: 1000000, cpuMs: undefined})
167
+ const current = sample({atMs: 1025000, cpuMs: undefined})
168
+ const result = classify({current, previous})
169
+
170
+ expect(result.status).toBe("unproven")
171
+ expect(result.cpuDeltaMs).toBe(undefined)
172
+ expect(result.reason).toBe("process CPU is not measurable on this host")
173
+ })
174
+
175
+ test("a generation with no live process left is not a wedge", () => {
176
+ const result = classify({current: sample({pid: undefined})})
177
+
178
+ expect(result.status).toBe("progressing")
179
+ expect(result.reason).toBe("no live process remains in the generation")
180
+ })
181
+
182
+ test("a process group that outlived its tracked process is still a live process of the generation", () => {
183
+ const result = classify({current: sample({heldProcessGroupPgid: 4242, liveChildCount: 0, pid: undefined})})
184
+
185
+ expect(result.status).toBe("wedged")
186
+ expect(result.reason).toBe("1 live process past its retirement budget with no runner child remaining; 1 tracked process already exited and left worker pgid 4242 alive")
187
+ })
188
+
189
+ test("a held process group that still consumes CPU across the window is a progressing drain", () => {
190
+ const previous = sample({atMs: 1000000, cpuMs: 5000, heldProcessGroupPgid: 4242, liveChildCount: 2, pid: undefined})
191
+ const current = sample({atMs: 1025000, cpuMs: 9000, heldProcessGroupPgid: 4242, liveChildCount: 2, pid: undefined})
192
+ const result = classify({current, previous})
193
+
194
+ expect(result.status).toBe("progressing")
195
+ expect(result.cpuDeltaMs).toBe(4000)
196
+ expect(result.reason).toBe("CPU advanced 4000ms across 25000ms")
197
+ })
198
+
199
+ test("a proven wedge with no runner child left is escalated, naming the process group it terminates", () => {
200
+ const previous = sample({atMs: 1000000, cpuMs: 5000, liveChildCount: 3})
201
+ const current = sample({atMs: 1025000, cpuMs: 5000, liveChildCount: 0})
202
+ const decision = escalate({current, previous})
203
+
204
+ expect(decision.decision).toBe("escalate")
205
+ expect(decision.killSafe).toBe(true)
206
+ expect(decision.targets).toEqual([{pgid: 4242, processId: "worker"}])
207
+ expect(decision.reason).toContain("CPU flat across 25000ms")
208
+ })
209
+
210
+ test("the escalation targets the group that outlived its tracked process", () => {
211
+ const previous = sample({atMs: 1000000, cpuMs: 5000, heldProcessGroupPgid: 4242, liveChildCount: 2, pid: undefined})
212
+ const current = sample({atMs: 1025000, cpuMs: 5000, heldProcessGroupPgid: 4242, liveChildCount: 0, pid: undefined})
213
+ const decision = escalate({current, previous})
214
+
215
+ expect(decision.decision).toBe("escalate")
216
+ expect(decision.targets).toEqual([{pgid: 4242, processId: "worker"}])
217
+ })
218
+
219
+ test("a wedge that still holds a live runner child is never escalated", () => {
220
+ const previous = sample({atMs: 1000000, cpuMs: 5000})
221
+ const current = sample({atMs: 1025000, cpuMs: 5000, liveChildCount: 3})
222
+
223
+ expect(classify({current, previous}).status).toBe("wedged")
224
+
225
+ const decision = escalate({current, previous})
226
+
227
+ expect(decision.decision).toBe("report")
228
+ expect(decision.killSafe).toBe(false)
229
+ expect(decision.targets).toEqual([])
230
+ expect(decision.reason).toBe("no escalation: 3 runner children are still live in the generation; SIGKILL would abort in-flight work")
231
+ })
232
+
233
+ test("a live process whose CPU is still moving is never escalated", () => {
234
+ const previous = sample({atMs: 1000000, cpuMs: 5000})
235
+ const current = sample({atMs: 1025000, cpuMs: 9000, liveChildCount: 0})
236
+
237
+ expect(classify({current, previous}).status).toBe("wedged")
238
+
239
+ const decision = escalate({current, previous})
240
+
241
+ expect(decision.decision).toBe("report")
242
+ expect(decision.reason).toBe("no escalation: CPU flatness is not proven (4000ms of CPU moved across 25000ms)")
243
+ })
244
+
245
+ test("a process that has not expired its retirement budget is never escalated", () => {
246
+ const previous = sample({atMs: 1000000, cpuMs: 5000, liveChildCount: 0, retirementTimedOut: false})
247
+ const current = sample({atMs: 1025000, cpuMs: 5000, liveChildCount: 0, retirementTimedOut: false})
248
+ const decision = escalate({current, previous})
249
+
250
+ expect(decision.decision).toBe("report")
251
+ expect(decision.reason).toBe("no escalation: drain is progressing (process worker is still inside its retirement budget)")
252
+ })
253
+
254
+ test("a generation that still owns a live connection is never escalated", () => {
255
+ const decision = escalate({connectionCount: 2, current: sample({liveChildCount: 0})})
256
+
257
+ expect(decision.decision).toBe("report")
258
+ expect(decision.reason).toBe("no escalation: drain is progressing (2 live connections still owned by the generation)")
259
+ })
260
+
261
+ test("a wedge with no measurable CPU is never escalated", () => {
262
+ const previous = sample({atMs: 1000000, cpuMs: undefined, liveChildCount: 0})
263
+ const current = sample({atMs: 1025000, cpuMs: undefined, liveChildCount: 0})
264
+
265
+ expect(classify({current, previous}).status).toBe("wedged")
266
+
267
+ const decision = escalate({current, previous})
268
+
269
+ expect(decision.decision).toBe("report")
270
+ expect(decision.reason).toBe("no escalation: CPU flatness is not proven (process CPU is not measurable on this host)")
271
+ })
272
+
273
+ test("a wedge seen in one observation is not escalated until a full window has passed", () => {
274
+ const current = sample({liveChildCount: 0})
275
+ const shortWindow = escalate({current: sample({atMs: 1001000, cpuMs: 5000, liveChildCount: 0}), previous: sample({atMs: 1000000, cpuMs: 5000, liveChildCount: 0})})
276
+
277
+ expect(classify({current}).status).toBe("wedged")
278
+ expect(escalate({current}).decision).toBe("report")
279
+ expect(escalate({current}).reason).toBe("no escalation: liveness window 0ms is below the 20000ms minimum")
280
+ expect(shortWindow.reason).toBe("no escalation: liveness window 1000ms is below the 20000ms minimum")
281
+ })
282
+
283
+ test("admission allows a handoff that stays within the bound and counts the incumbent", () => {
284
+ const admission = admitGenerationRelease({
285
+ candidateReleaseId: "v3",
286
+ draining: [{reason: "1 live connection still owned by the generation", releaseId: "v1", status: "progressing"}],
287
+ incumbentWillDrain: true,
288
+ maxConcurrentDrainingReleaseGroups: 2
289
+ })
290
+
291
+ expect(admission.allowed).toBe(true)
292
+ expect(admission.wouldBeDrainingCount).toBe(2)
293
+ expect(admission.reason).toBe(undefined)
294
+ })
295
+
296
+ test("admission refuses before activation and names every draining group with its classification", () => {
297
+ const admission = admitGenerationRelease({
298
+ candidateReleaseId: "20261115073717",
299
+ draining: [
300
+ {reason: "1 live process with 3 runner children, no live connection, every retirement budget expired, CPU flat across 25000ms", releaseId: "20261115063805", status: "wedged"},
301
+ {reason: "2 live connections still owned by the generation", releaseId: "20261114232900", status: "progressing"}
302
+ ],
303
+ incumbentWillDrain: true,
304
+ maxConcurrentDrainingReleaseGroups: 2
305
+ })
306
+
307
+ expect(admission.allowed).toBe(false)
308
+ expect(admission.wouldBeDrainingCount).toBe(3)
309
+ expect(admission.reason).toContain("Deploy refused before activation: 2 release groups already draining")
310
+ expect(admission.reason).toContain("20261115063805 wedged")
311
+ expect(admission.reason).toContain("Activating 20261115073717 would make 3, above the releaseRetention.maxConcurrentDrainingReleaseGroups bound of 2")
312
+ })
313
+
314
+ test("a first deploy with no incumbent adds no draining group", () => {
315
+ const admission = admitGenerationRelease({candidateReleaseId: "v1", draining: [], incumbentWillDrain: false, maxConcurrentDrainingReleaseGroups: 1})
316
+
317
+ expect(admission.allowed).toBe(true)
318
+ expect(admission.wouldBeDrainingCount).toBe(0)
319
+ })
320
+
321
+ test("a bound of 0 disables the admission check", () => {
322
+ const draining = Array.from({length: 30}, (_, index) => ({reason: "no runner child remaining", releaseId: `v${index}`, status: /** @type {const} */ ("wedged")}))
323
+ const admission = admitGenerationRelease({candidateReleaseId: "v31", draining, incumbentWillDrain: true, maxConcurrentDrainingReleaseGroups: 0})
324
+
325
+ expect(admission.allowed).toBe(true)
326
+ expect(admission.wouldBeDrainingCount).toBe(31)
327
+ })
328
+ })
@@ -0,0 +1,106 @@
1
+ // @ts-check
2
+
3
+ import http from "node:http"
4
+ import net from "node:net"
5
+
6
+ const port = Number(requiredEnv("ROLLBRIDGE_PORT"))
7
+ const bootMs = Number(process.env.ROLLBRIDGE_BOOT_MS || "0")
8
+ const dependencyPort = process.env.ROLLBRIDGE_DEPENDENCY_PORT
9
+ const releaseId = process.env.ROLLBRIDGE_RELEASE_ID || "unknown"
10
+
11
+ log(`starting (boot budget ${bootMs}ms)`)
12
+
13
+ if (dependencyPort) {
14
+ // The app performs its own dependency wait, exactly as the production web command does.
15
+ await waitForDependency(Number(dependencyPort))
16
+ log(`dependency ${dependencyPort} accepted a connection`)
17
+ }
18
+
19
+ await sleep(bootMs)
20
+
21
+ const server = http.createServer((request, response) => {
22
+ if (request.url === "/ping") {
23
+ response.writeHead(200, {"Content-Type": "application/json"})
24
+ response.end(JSON.stringify({message: "Pong", releaseId}))
25
+ return
26
+ }
27
+
28
+ response.writeHead(200, {"Content-Type": "text/plain; charset=utf-8"})
29
+ response.end(`${releaseId}\n`)
30
+ })
31
+
32
+ process.on("SIGTERM", () => {
33
+ server.close(() => process.exit(0))
34
+ setTimeout(() => process.exit(0), 10).unref()
35
+ })
36
+
37
+ server.listen(port, "127.0.0.1")
38
+ log(`listening on ${port}`)
39
+
40
+ /**
41
+ * @param {number} dependencyPortNumber - Port of the process this app depends on.
42
+ * @returns {Promise<void>} Resolves once the dependency accepts a connection.
43
+ */
44
+ async function waitForDependency(dependencyPortNumber) {
45
+ const deadline = Date.now() + 10000
46
+
47
+ while (Date.now() < deadline) {
48
+ if (await tryConnect(dependencyPortNumber)) return
49
+ await sleep(50)
50
+ }
51
+
52
+ log(`dependency ${dependencyPortNumber} never accepted a connection`)
53
+ process.exit(21)
54
+ }
55
+
56
+ /**
57
+ * @param {number} dependencyPortNumber - Port to connect to.
58
+ * @returns {Promise<boolean>} True when the dependency accepted the connection.
59
+ */
60
+ function tryConnect(dependencyPortNumber) {
61
+ return new Promise((resolve) => {
62
+ const socket = net.createConnection({host: "127.0.0.1", port: dependencyPortNumber})
63
+ let settled = false
64
+
65
+ /** @param {boolean} connected - Whether the connection succeeded. */
66
+ const finish = (connected) => {
67
+ if (settled) return
68
+ settled = true
69
+ socket.destroy()
70
+ resolve(connected)
71
+ }
72
+
73
+ socket.setTimeout(200)
74
+ socket.once("connect", () => finish(true))
75
+ socket.once("error", () => finish(false))
76
+ socket.once("timeout", () => finish(false))
77
+ })
78
+ }
79
+
80
+ /**
81
+ * @param {number} ms - Milliseconds.
82
+ * @returns {Promise<void>} Resolves after the delay.
83
+ */
84
+ function sleep(ms) {
85
+ return new Promise((resolve) => setTimeout(resolve, ms))
86
+ }
87
+
88
+ /**
89
+ * @param {string} line - Output line, so the failure record can name the app's last output.
90
+ * @returns {void}
91
+ */
92
+ function log(line) {
93
+ console.log(`boot: ${process.pid} ${line}`)
94
+ }
95
+
96
+ /**
97
+ * @param {string} key - Environment variable name.
98
+ * @returns {string} Environment variable value.
99
+ */
100
+ function requiredEnv(key) {
101
+ const value = process.env[key]
102
+
103
+ if (!value) throw new Error(`${key} is required`)
104
+
105
+ return value
106
+ }
@@ -0,0 +1,50 @@
1
+ // @ts-check
2
+
3
+ import http from "node:http"
4
+ import fs from "node:fs"
5
+
6
+ const port = Number(requiredEnv("ROLLBRIDGE_PORT"))
7
+ const gateArg = process.argv.find((arg) => arg.startsWith("--gate="))
8
+
9
+ if (!gateArg) throw new Error("--gate=<path> is required")
10
+
11
+ const gatePath = gateArg.slice("--gate=".length)
12
+ const releaseId = process.env.ROLLBRIDGE_RELEASE_ID || "unknown"
13
+ // v3 stays unhealthy until the gate file appears, so its deploy fails at health.
14
+ const unhealthy = releaseId === "v3" && !fs.existsSync(gatePath)
15
+
16
+ const server = http.createServer((request, response) => {
17
+ if (unhealthy) {
18
+ response.writeHead(500)
19
+ response.end("bad")
20
+ return
21
+ }
22
+
23
+ if (request.url === "/ping") {
24
+ response.writeHead(200, {"Content-Type": "application/json"})
25
+ response.end(JSON.stringify({message: "Pong", releaseId}))
26
+ return
27
+ }
28
+
29
+ response.writeHead(200, {"Content-Type": "text/plain; charset=utf-8"})
30
+ response.end(`${releaseId}\n`)
31
+ })
32
+
33
+ process.on("SIGTERM", () => {
34
+ server.close(() => process.exit(0))
35
+ setTimeout(() => process.exit(0), 10).unref()
36
+ })
37
+
38
+ server.listen(port, "127.0.0.1")
39
+
40
+ /**
41
+ * @param {string} key - Environment variable name.
42
+ * @returns {string} Environment variable value.
43
+ */
44
+ function requiredEnv(key) {
45
+ const value = process.env[key]
46
+
47
+ if (!value) throw new Error(`${key} is required`)
48
+
49
+ return value
50
+ }
@@ -7,19 +7,39 @@ import os from "node:os"
7
7
  import path from "node:path"
8
8
  import {describe, expect, test} from "@velocious/testing"
9
9
  import {fileURLToPath} from "node:url"
10
- import GuardianClient from "../src/guardian-client.js"
10
+ import {createRequire} from "node:module"
11
+ import GuardianClient, {usableSocketPath} from "../src/guardian-client.js"
11
12
  import {waitForProcessExit} from "./support/process.js"
12
13
 
13
14
  describe("guardian-client", () => {
14
15
 
16
+ const commonJsRequire = createRequire(import.meta.url)
17
+ const packageVersion = /** @type {{version: string}} */ (commonJsRequire("../package.json")).version
15
18
  const legacyGuardianPath = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures", "pre-split3-process-guardian.js")
16
19
  const recoveryOwnerPath = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures", "guardian-recovery-owner.js")
17
20
 
21
+ test("guardian maps an overlong configured socket path to a stable short path", async () => {
22
+ const root = await fs.mkdtemp(path.join(os.tmpdir(), `rollbridge-${"x".repeat(120)}-`))
23
+ const configuredPath = path.join(root, "guardian.sock")
24
+ const client = new GuardianClient({socketPath: configuredPath, token: "long-path-capability"})
25
+
26
+ try {
27
+ expect(Buffer.byteLength(configuredPath)).toBeGreaterThan(107)
28
+ expect(Buffer.byteLength(client.socketPath)).toBeLessThanOrEqual(107)
29
+ expect(client.socketPath).not.toBe(configuredPath)
30
+ await client.launch()
31
+ expect(await client.capabilities()).toEqual({daemonRecovery: 1, generationReactivation: 1, guardianVersion: packageVersion})
32
+ } finally {
33
+ await cleanupGuardian({client, root})
34
+ }
35
+ })
36
+
18
37
  test("guardian bootstrap capability is absent from process argv", async () => {
19
38
  const fixture = await createGuardian()
39
+ const packageVersion = JSON.parse(await fs.readFile(path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "package.json"), "utf8")).version
20
40
 
21
41
  try {
22
- expect(await fixture.client.capabilities()).toEqual({daemonRecovery: 1, generationReactivation: 1})
42
+ expect(await fixture.client.capabilities()).toEqual({daemonRecovery: 1, generationReactivation: 1, guardianVersion: packageVersion})
23
43
  const commandLine = await fs.readFile(`/proc/${fixture.client.pid}/cmdline`, "utf8")
24
44
  const environment = await fs.readFile(`/proc/${fixture.client.pid}/environ`, "utf8")
25
45
  const status = await fs.readFile(`/proc/${fixture.client.pid}/status`, "utf8")
@@ -58,6 +78,100 @@ test("guardian inventory removes only an exact owned provenance", async () => {
58
78
  }
59
79
  })
60
80
 
81
+ test("a refused owner mutation names the held mutation, its process, its age and its blocker", async () => {
82
+ const fixture = await createGuardian()
83
+
84
+ try {
85
+ const mutationId = await fixture.client.beginOwnerMutation("restart", {ownerPid: process.pid, waitingFor: "the managed process to stop and start again"})
86
+ await expect(fixture.client.beginOwnerMutation("deploy")).rejects.toThrow(/^Another owner mutation is already in progress: owner mutation "restart"; mutation [0-9a-f-]{36}; held by process \d+; since 20\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\.\d\d\dZ; \d+s elapsed; waiting for the managed process to stop and start again; refused deploy$/)
87
+ await fixture.client.reportOwnerMutationWait(mutationId, "an idle forked runner that never exits")
88
+ await expect(fixture.client.beginOwnerMutation("deploy")).rejects.toThrow(/waiting for an idle forked runner that never exits/)
89
+ await fixture.client.endOwnerMutation(mutationId)
90
+ expect(typeof await fixture.client.beginOwnerMutation("deploy")).toBe("string")
91
+ } finally {
92
+ await cleanupGuardian(fixture)
93
+ }
94
+ })
95
+
96
+ test("an owner mutation that outruns its lease is released and the holder reports the expiry", async () => {
97
+ const fixture = await createGuardian()
98
+
99
+ try {
100
+ const wedged = await fixture.client.beginOwnerMutation("restart", {mutationTimeoutMs: 200, ownerPid: process.pid, waitingFor: "a drain that never returns"})
101
+
102
+ await new Promise((resolve) => setTimeout(resolve, 600))
103
+ const successor = await fixture.client.beginOwnerMutation("deploy", {ownerPid: process.pid, waitingFor: "candidate activation"})
104
+
105
+ expect(typeof successor).toBe("string")
106
+ await expect(fixture.client.endOwnerMutation(wedged)).rejects.toThrow(/^Owner mutation restart exceeded its 200 ms lease and was released after \d+ ms; the mutation failed and its lease is gone$/)
107
+ await fixture.client.endOwnerMutation(successor)
108
+ } finally {
109
+ await cleanupGuardian(fixture)
110
+ }
111
+ })
112
+
113
+ test("a dead owner-mutation holder is reclaimed and the next mutation is admitted", async () => {
114
+ const fixture = await createGuardian()
115
+ const holder = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"])
116
+
117
+ try {
118
+ const reclaimed = fixture.client.waitForEvent("owner-mutation-reclaimed")
119
+ await fixture.client.beginOwnerMutation("restart", {ownerPid: holder.pid, waitingFor: "a process that will die"})
120
+ if (!holder.pid) throw new Error("Missing mutation holder PID")
121
+ holder.kill("SIGKILL")
122
+ await once(holder, "exit")
123
+
124
+ const successor = await fixture.client.beginOwnerMutation("deploy", {ownerPid: process.pid, waitingFor: "candidate activation"})
125
+ const event = await reclaimed
126
+
127
+ expect(typeof successor).toBe("string")
128
+ expect(event).toEqual(expect.objectContaining({code: "OWNER_MUTATION_RECLAIMED", event: "owner-mutation-reclaimed", operation: "restart", pid: holder.pid, reason: "owner mutation holder process is no longer alive"}))
129
+ expect((await fixture.client.guardianStatus()).version).toBe(packageVersion)
130
+ await fixture.client.endOwnerMutation(successor)
131
+ } finally {
132
+ if (holder.exitCode === null && holder.signalCode === null) holder.kill("SIGKILL")
133
+ await cleanupGuardian(fixture)
134
+ }
135
+ })
136
+
137
+ test("guardian lets the owner re-register a key whose registered child is gone, and still fences a live one", async () => {
138
+ const fixture = await createGuardian()
139
+ const liveInstance = fixture.client.process("live-generation", definition("live-generation"))
140
+ const deadInstance = fixture.client.process("dead-generation", definition("dead-generation"))
141
+
142
+ try {
143
+ await liveInstance.start()
144
+ await deadInstance.start()
145
+ const deadPid = (await deadInstance.status()).pid
146
+
147
+ if (!deadPid) throw new Error("Missing required fixture: dead-generation pid")
148
+ await deadInstance.stop()
149
+ await waitForProcessExit(deadPid)
150
+ const deadStatus = await deadInstance.status()
151
+
152
+ expect(deadStatus.state).toBe("stopped")
153
+ expect(deadStatus.pid).toBeUndefined()
154
+
155
+ // The wedge this guards against: a later deploy reconstructs the same key from a
156
+ // different release definition. The old child is conclusively gone, so the old
157
+ // provenance must not fence the new registration.
158
+ const nextInstance = fixture.client.process("dead-generation", {...definition("dead-generation"), env: {REPLACED_BY_LATER_DEPLOY: "1"}})
159
+ const reRegistered = nextInstance.status()
160
+
161
+ expect(reRegistered.state).toBe("stopped")
162
+ await nextInstance.start()
163
+ expect((await nextInstance.status()).state).toBe("running")
164
+ expect((await fixture.client.inventory()).some((entry) => entry.key === "dead-generation" && entry.provenance === nextInstance.provenance)).toBe(true)
165
+
166
+ // A live process with a different provenance stays fail-closed.
167
+ const hostileInstance = fixture.client.process("live-generation", {...definition("live-generation"), env: {HOSTILE: "1"}})
168
+
169
+ await expect(hostileInstance.start()).rejects.toThrow(/provenance mismatch/)
170
+ } finally {
171
+ await cleanupGuardian(fixture)
172
+ }
173
+ })
174
+
61
175
  test("guardian runs a strict activation lifecycle command for the exact registered process", async () => {
62
176
  const fixture = await createGuardian()
63
177
  const activationPath = path.join(fixture.root, "activated")
@@ -842,7 +956,7 @@ test("replacement staging rejects owner state published after prepare", async ()
842
956
  const prepared = await candidate.prepareOwnerReplacement(authority, nextAuthority)
843
957
 
844
958
  await fixture.client.publishOwnerState({authority, snapshot: {activeReleaseId: "v2"}})
845
- await expect(candidate.stageOwnerReplacement(prepared.replacementId, {authority: nextAuthority, snapshot: {activeReleaseId: "v1"}})).rejects.toThrow(/owner state changed after prepare/i)
959
+ await expect(candidate.stageOwnerReplacement(prepared.replacementId, {authority: nextAuthority, snapshot: {activeReleaseId: "v1"}})).rejects.toThrow(/owner state changed after prepare \(prepared revision 1, current revision 2\)/i)
846
960
  await candidate.abortOwnerReplacement(prepared.replacementId)
847
961
  const fresh = await candidate.prepareOwnerReplacement(authority, nextAuthority)
848
962
 
@@ -1678,7 +1792,7 @@ test("a legacy bridge remains discoverable when its candidate disconnects after
1678
1792
  let replacement
1679
1793
 
1680
1794
  try {
1681
- const identity = {socketPath: path.join(fixture.root, "guardian-v2.sock"), token: "candidate-guardian-capability"}
1795
+ const identity = {socketPath: usableSocketPath(path.join(fixture.root, "guardian-v2.sock")), token: "candidate-guardian-capability"}
1682
1796
 
1683
1797
  upgraded = await fixture.client.upgradeLegacyGuardian({ownerState, ...identity})
1684
1798
  const prepared = await upgraded.prepareOwnerReplacement(authority, nextAuthority)
@@ -1725,7 +1839,7 @@ async function createGuardian() {
1725
1839
  async function createLegacyGuardian() {
1726
1840
  const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-legacy-guardian-"))
1727
1841
  const token = "legacy-guardian-capability"
1728
- const socketPath = path.join(root, "guardian.sock")
1842
+ const socketPath = usableSocketPath(path.join(root, "guardian.sock"))
1729
1843
  const child = spawn(process.execPath, [legacyGuardianPath, socketPath], {detached: true, stdio: ["ignore", "ignore", "ignore", "ipc"]})
1730
1844
 
1731
1845
  await new Promise((resolve, reject) => {