rollbridge 0.1.72 → 0.1.74
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +63 -10
- package/TODO.md +13 -0
- package/changelog.d/20261025-guardian-re-register-dead-generation.md +13 -0
- package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +1 -0
- package/changelog.d/20261104-compose-service-units.md +21 -0
- package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +20 -0
- package/changelog.d/20261106-release-dependency-boot-budgets.md +32 -0
- package/changelog.d/20261109-path-filtered-builds.md +11 -0
- package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +26 -0
- package/changelog.d/20261111-contention-sensitive-daemon-specs.md +27 -0
- package/changelog.d/20261113-bounded-nonblocking-retirement.md +5 -0
- package/changelog.d/20261115-bound-draining-release-groups.md +62 -0
- package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +27 -0
- package/changelog.d/20261115-wedged-generation-escalation.md +44 -0
- package/changelog.d/20261116-bounded-explicit-stop-retirement.md +30 -0
- package/changelog.d/20261116-config-json-canonical-shape.md +11 -0
- package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +29 -0
- package/changelog.d/20261116-guardian-version-handoff.md +37 -0
- package/changelog.d/20261116-reclaim-dead-owner-mutation.md +7 -0
- package/changelog.d/20261117-activate-before-incumbent-retirement.md +1 -0
- package/changelog.d/20261117-activation-path-drain-deadline.md +35 -0
- package/changelog.d/20261117-stop-companion-after-retired-dependency.md +7 -0
- package/changelog.d/20261118-incumbent-retirement-authority-guard.md +1 -0
- package/changelog.d/20261118-owner-retirement-no-guardian-bound.md +19 -0
- package/changelog.d/20261118-shutdown-retirement-bound-restored.md +31 -0
- package/docs/cli.md +128 -3
- package/docs/compose-services.md +222 -0
- package/docs/config.md +177 -6
- package/docs/troubleshooting.md +97 -6
- package/docs/velocious.md +1 -1
- package/docs/workers.md +11 -0
- package/examples/tensorbuzz.com.js +3 -2
- package/package.json +1 -1
- package/src/cli.js +68 -2
- package/src/compose-unit.js +725 -0
- package/src/config.js +346 -8
- package/src/daemon.js +410 -73
- package/src/doctor.js +74 -0
- package/src/drain-policy.js +252 -0
- package/src/guardian-client.js +67 -8
- package/src/health.js +99 -8
- package/src/managed-process.js +197 -16
- package/src/process-guardian.js +212 -22
- package/src/process-memory.js +62 -2
- package/src/process-order.js +26 -0
- package/src/release-group.js +435 -37
- package/tensorbuzz.yml +20 -1
- package/test/completion.test.js +5 -2
- package/test/compose-config.test.js +160 -0
- package/test/compose-docker.test.js +297 -0
- package/test/compose-unit.test.js +415 -0
- package/test/config-validation.test.js +135 -5
- package/test/drain-policy.test.js +328 -0
- package/test/fixtures/deferred-boot-app.js +106 -0
- package/test/fixtures/gated-app.js +50 -0
- package/test/guardian-client.test.js +119 -5
- package/test/health.test.js +74 -1
- package/test/managed-process.test.js +260 -1
- package/test/owner-recovery.test.js +125 -21
- package/test/owner-replacement.test.js +175 -14
- package/test/process-memory.test.js +47 -1
- package/test/release-group.test.js +408 -2
- package/test/rollbridge.test.js +774 -72
- package/test/shutdown-completion.test.js +119 -0
- package/test/shutdown-retirement-bound.test.js +162 -0
- package/test/support/process.js +3 -1
- package/test/tensorbuzz-config.test.js +228 -0
|
@@ -0,0 +1,328 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
import {describe, expect, test} from "@velocious/testing"
|
|
4
|
+
import {admitGenerationRelease, classifyDrainProgress, decideWedgedEscalation} from "../src/drain-policy.js"
|
|
5
|
+
|
|
6
|
+
describe("drain-policy", () => {
|
|
7
|
+
|
|
8
|
+
const WINDOW_MS = 20000
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* @param {object} [options] - Observation shape.
|
|
12
|
+
* @param {number} [options.atMs] - Observation time.
|
|
13
|
+
* @param {number | undefined} [options.cpuMs] - Cumulative CPU of the group.
|
|
14
|
+
* @param {number | undefined} [options.heldProcessGroupPgid] - Group still alive after its tracked process exited.
|
|
15
|
+
* @param {number} [options.liveChildCount] - Runner children other than the group leader.
|
|
16
|
+
* @param {number | undefined} [options.pid] - Live pid, or undefined when the process is gone.
|
|
17
|
+
* @param {boolean} [options.retirementTimedOut] - Whether the supervisor already gave up on it.
|
|
18
|
+
* @returns {import("../src/drain-policy.js").DrainLivenessSample} One liveness observation.
|
|
19
|
+
*/
|
|
20
|
+
function sample(options = {}) {
|
|
21
|
+
return {
|
|
22
|
+
atMs: options.atMs ?? 1000000,
|
|
23
|
+
processes: [{
|
|
24
|
+
cpuMs: "cpuMs" in options ? options.cpuMs : 5000,
|
|
25
|
+
heldProcessGroupPgid: options.heldProcessGroupPgid,
|
|
26
|
+
id: "worker",
|
|
27
|
+
liveChildCount: options.liveChildCount ?? 1,
|
|
28
|
+
pid: "pid" in options ? options.pid : 4242,
|
|
29
|
+
retirementTimedOut: options.retirementTimedOut ?? true
|
|
30
|
+
}]
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* @param {object} [options] - Classification inputs.
|
|
36
|
+
* @param {number} [options.connectionCount] - Live connections owned by the generation.
|
|
37
|
+
* @param {import("../src/drain-policy.js").DrainLivenessSample} [options.current] - Newest observation.
|
|
38
|
+
* @param {number} [options.drainBudgetMs] - Configured drain budget.
|
|
39
|
+
* @param {number} [options.drainElapsedMs] - Time since the drain started.
|
|
40
|
+
* @param {import("../src/drain-policy.js").DrainLivenessSample | undefined} [options.previous] - Older observation.
|
|
41
|
+
* @returns {import("../src/drain-policy.js").DrainClassification} Classification.
|
|
42
|
+
*/
|
|
43
|
+
function classify(options = {}) {
|
|
44
|
+
return classifyDrainProgress({
|
|
45
|
+
connectionCount: options.connectionCount ?? 0,
|
|
46
|
+
currentSample: options.current ?? sample(),
|
|
47
|
+
drainBudgetMs: options.drainBudgetMs ?? 0,
|
|
48
|
+
drainElapsedMs: options.drainElapsedMs ?? 0,
|
|
49
|
+
minSampleWindowMs: WINDOW_MS,
|
|
50
|
+
previousSample: options.previous
|
|
51
|
+
})
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* @param {object} [options] - Decision inputs.
|
|
56
|
+
* @param {number} [options.connectionCount] - Live connections owned by the generation.
|
|
57
|
+
* @param {import("../src/drain-policy.js").DrainLivenessSample} [options.current] - Newest observation.
|
|
58
|
+
* @param {import("../src/drain-policy.js").DrainLivenessSample | undefined} [options.previous] - Older observation.
|
|
59
|
+
* @returns {import("../src/drain-policy.js").EscalationDecision} Escalation decision for the same pair the classifier saw.
|
|
60
|
+
*/
|
|
61
|
+
function escalate(options = {}) {
|
|
62
|
+
const currentSample = options.current ?? sample()
|
|
63
|
+
|
|
64
|
+
return decideWedgedEscalation({
|
|
65
|
+
classification: classify({connectionCount: options.connectionCount, current: currentSample, previous: options.previous}),
|
|
66
|
+
currentSample,
|
|
67
|
+
minSampleWindowMs: WINDOW_MS
|
|
68
|
+
})
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
test("a generation that still owns connections is draining, never wedged", () => {
|
|
72
|
+
const result = classify({connectionCount: 3, current: sample({liveChildCount: 0})})
|
|
73
|
+
|
|
74
|
+
expect(result.status).toBe("progressing")
|
|
75
|
+
expect(result.reason).toBe("3 live connections still owned by the generation")
|
|
76
|
+
})
|
|
77
|
+
|
|
78
|
+
test("a live connection past the configured drain budget is reported past the deadline, not progressing", () => {
|
|
79
|
+
const result = classify({connectionCount: 1, current: sample({liveChildCount: 0}), drainBudgetMs: 60000, drainElapsedMs: 61000})
|
|
80
|
+
|
|
81
|
+
expect(result.status).toBe("timedOut")
|
|
82
|
+
expect(result.reason).toBe("drain budget 60000ms exceeded by 1000ms with 1 live connection still owned by the generation")
|
|
83
|
+
})
|
|
84
|
+
|
|
85
|
+
test("a live connection inside the configured drain budget is still a legitimate drain", () => {
|
|
86
|
+
const result = classify({connectionCount: 1, current: sample({liveChildCount: 0}), drainBudgetMs: 60000, drainElapsedMs: 30000})
|
|
87
|
+
|
|
88
|
+
expect(result.status).toBe("progressing")
|
|
89
|
+
expect(result.reason).toBe("1 live connection still owned by the generation")
|
|
90
|
+
})
|
|
91
|
+
|
|
92
|
+
test("the drain deadline names the deadline and never escalates", () => {
|
|
93
|
+
const currentSample = sample({liveChildCount: 0})
|
|
94
|
+
const classification = classify({connectionCount: 1, current: currentSample, drainBudgetMs: 60000, drainElapsedMs: 61000})
|
|
95
|
+
const decision = decideWedgedEscalation({classification, currentSample, minSampleWindowMs: WINDOW_MS})
|
|
96
|
+
|
|
97
|
+
expect(decision.decision).toBe("report")
|
|
98
|
+
expect(decision.reason).toBe("no escalation: drain is timedOut (drain budget 60000ms exceeded by 1000ms with 1 live connection still owned by the generation)")
|
|
99
|
+
expect(decision.targets).toEqual([])
|
|
100
|
+
})
|
|
101
|
+
|
|
102
|
+
test("a process still inside its retirement budget is draining", () => {
|
|
103
|
+
const result = classify({current: sample({liveChildCount: 0, retirementTimedOut: false})})
|
|
104
|
+
|
|
105
|
+
expect(result.status).toBe("progressing")
|
|
106
|
+
expect(result.reason).toBe("process worker is still inside its retirement budget")
|
|
107
|
+
})
|
|
108
|
+
|
|
109
|
+
test("a live process past its retirement budget with no runner child left is wedged", () => {
|
|
110
|
+
const result = classify({current: sample({liveChildCount: 0})})
|
|
111
|
+
|
|
112
|
+
expect(result.status).toBe("wedged")
|
|
113
|
+
expect(result.reason).toBe("1 live process past its retirement budget with no runner child remaining")
|
|
114
|
+
})
|
|
115
|
+
|
|
116
|
+
test("flat CPU across a sufficient window is wedged, and the window is reported", () => {
|
|
117
|
+
const previous = sample({atMs: 1000000, cpuMs: 5000})
|
|
118
|
+
const current = sample({atMs: 1025000, cpuMs: 5000, liveChildCount: 3})
|
|
119
|
+
const result = classify({current, previous})
|
|
120
|
+
|
|
121
|
+
expect(result.status).toBe("wedged")
|
|
122
|
+
expect(result.sampleWindowMs).toBe(25000)
|
|
123
|
+
expect(result.cpuDeltaMs).toBe(0)
|
|
124
|
+
expect(result.reason).toBe("1 live process with 3 runner children, no live connection, every retirement budget expired, CPU flat across 25000ms")
|
|
125
|
+
})
|
|
126
|
+
|
|
127
|
+
test("CPU that advanced across the window is a progressing drain", () => {
|
|
128
|
+
const previous = sample({atMs: 1000000, cpuMs: 5000})
|
|
129
|
+
const current = sample({atMs: 1025000, cpuMs: 9000})
|
|
130
|
+
const result = classify({current, previous})
|
|
131
|
+
|
|
132
|
+
expect(result.status).toBe("progressing")
|
|
133
|
+
expect(result.cpuDeltaMs).toBe(4000)
|
|
134
|
+
expect(result.reason).toBe("CPU advanced 4000ms across 25000ms")
|
|
135
|
+
})
|
|
136
|
+
|
|
137
|
+
test("CPU leaving the group with an exited member is a progressing drain", () => {
|
|
138
|
+
const previous = sample({atMs: 1000000, cpuMs: 9000})
|
|
139
|
+
const current = sample({atMs: 1025000, cpuMs: 5000})
|
|
140
|
+
const result = classify({current, previous})
|
|
141
|
+
|
|
142
|
+
expect(result.status).toBe("progressing")
|
|
143
|
+
expect(result.cpuDeltaMs).toBe(-4000)
|
|
144
|
+
expect(result.reason).toBe("the group changed (1 live process, -4000ms of consumed CPU left the group) across 25000ms")
|
|
145
|
+
})
|
|
146
|
+
|
|
147
|
+
test("the first observation cannot claim flatness", () => {
|
|
148
|
+
const result = classify({current: sample()})
|
|
149
|
+
|
|
150
|
+
expect(result.status).toBe("unproven")
|
|
151
|
+
expect(result.sampleWindowMs).toBe(0)
|
|
152
|
+
expect(result.reason).toBe("first liveness observation; a 20000ms window is required before CPU flatness can be claimed")
|
|
153
|
+
})
|
|
154
|
+
|
|
155
|
+
test("a window shorter than the minimum cannot claim flatness", () => {
|
|
156
|
+
const previous = sample({atMs: 1000000})
|
|
157
|
+
const current = sample({atMs: 1010000})
|
|
158
|
+
const result = classify({current, previous})
|
|
159
|
+
|
|
160
|
+
expect(result.status).toBe("unproven")
|
|
161
|
+
expect(result.sampleWindowMs).toBe(10000)
|
|
162
|
+
expect(result.reason).toBe("liveness window 10000ms is below the 20000ms minimum")
|
|
163
|
+
})
|
|
164
|
+
|
|
165
|
+
test("an unmeasurable CPU counter is unproven, never wedged", () => {
|
|
166
|
+
const previous = sample({atMs: 1000000, cpuMs: undefined})
|
|
167
|
+
const current = sample({atMs: 1025000, cpuMs: undefined})
|
|
168
|
+
const result = classify({current, previous})
|
|
169
|
+
|
|
170
|
+
expect(result.status).toBe("unproven")
|
|
171
|
+
expect(result.cpuDeltaMs).toBe(undefined)
|
|
172
|
+
expect(result.reason).toBe("process CPU is not measurable on this host")
|
|
173
|
+
})
|
|
174
|
+
|
|
175
|
+
test("a generation with no live process left is not a wedge", () => {
|
|
176
|
+
const result = classify({current: sample({pid: undefined})})
|
|
177
|
+
|
|
178
|
+
expect(result.status).toBe("progressing")
|
|
179
|
+
expect(result.reason).toBe("no live process remains in the generation")
|
|
180
|
+
})
|
|
181
|
+
|
|
182
|
+
test("a process group that outlived its tracked process is still a live process of the generation", () => {
|
|
183
|
+
const result = classify({current: sample({heldProcessGroupPgid: 4242, liveChildCount: 0, pid: undefined})})
|
|
184
|
+
|
|
185
|
+
expect(result.status).toBe("wedged")
|
|
186
|
+
expect(result.reason).toBe("1 live process past its retirement budget with no runner child remaining; 1 tracked process already exited and left worker pgid 4242 alive")
|
|
187
|
+
})
|
|
188
|
+
|
|
189
|
+
test("a held process group that still consumes CPU across the window is a progressing drain", () => {
|
|
190
|
+
const previous = sample({atMs: 1000000, cpuMs: 5000, heldProcessGroupPgid: 4242, liveChildCount: 2, pid: undefined})
|
|
191
|
+
const current = sample({atMs: 1025000, cpuMs: 9000, heldProcessGroupPgid: 4242, liveChildCount: 2, pid: undefined})
|
|
192
|
+
const result = classify({current, previous})
|
|
193
|
+
|
|
194
|
+
expect(result.status).toBe("progressing")
|
|
195
|
+
expect(result.cpuDeltaMs).toBe(4000)
|
|
196
|
+
expect(result.reason).toBe("CPU advanced 4000ms across 25000ms")
|
|
197
|
+
})
|
|
198
|
+
|
|
199
|
+
test("a proven wedge with no runner child left is escalated, naming the process group it terminates", () => {
|
|
200
|
+
const previous = sample({atMs: 1000000, cpuMs: 5000, liveChildCount: 3})
|
|
201
|
+
const current = sample({atMs: 1025000, cpuMs: 5000, liveChildCount: 0})
|
|
202
|
+
const decision = escalate({current, previous})
|
|
203
|
+
|
|
204
|
+
expect(decision.decision).toBe("escalate")
|
|
205
|
+
expect(decision.killSafe).toBe(true)
|
|
206
|
+
expect(decision.targets).toEqual([{pgid: 4242, processId: "worker"}])
|
|
207
|
+
expect(decision.reason).toContain("CPU flat across 25000ms")
|
|
208
|
+
})
|
|
209
|
+
|
|
210
|
+
test("the escalation targets the group that outlived its tracked process", () => {
|
|
211
|
+
const previous = sample({atMs: 1000000, cpuMs: 5000, heldProcessGroupPgid: 4242, liveChildCount: 2, pid: undefined})
|
|
212
|
+
const current = sample({atMs: 1025000, cpuMs: 5000, heldProcessGroupPgid: 4242, liveChildCount: 0, pid: undefined})
|
|
213
|
+
const decision = escalate({current, previous})
|
|
214
|
+
|
|
215
|
+
expect(decision.decision).toBe("escalate")
|
|
216
|
+
expect(decision.targets).toEqual([{pgid: 4242, processId: "worker"}])
|
|
217
|
+
})
|
|
218
|
+
|
|
219
|
+
test("a wedge that still holds a live runner child is never escalated", () => {
|
|
220
|
+
const previous = sample({atMs: 1000000, cpuMs: 5000})
|
|
221
|
+
const current = sample({atMs: 1025000, cpuMs: 5000, liveChildCount: 3})
|
|
222
|
+
|
|
223
|
+
expect(classify({current, previous}).status).toBe("wedged")
|
|
224
|
+
|
|
225
|
+
const decision = escalate({current, previous})
|
|
226
|
+
|
|
227
|
+
expect(decision.decision).toBe("report")
|
|
228
|
+
expect(decision.killSafe).toBe(false)
|
|
229
|
+
expect(decision.targets).toEqual([])
|
|
230
|
+
expect(decision.reason).toBe("no escalation: 3 runner children are still live in the generation; SIGKILL would abort in-flight work")
|
|
231
|
+
})
|
|
232
|
+
|
|
233
|
+
test("a live process whose CPU is still moving is never escalated", () => {
|
|
234
|
+
const previous = sample({atMs: 1000000, cpuMs: 5000})
|
|
235
|
+
const current = sample({atMs: 1025000, cpuMs: 9000, liveChildCount: 0})
|
|
236
|
+
|
|
237
|
+
expect(classify({current, previous}).status).toBe("wedged")
|
|
238
|
+
|
|
239
|
+
const decision = escalate({current, previous})
|
|
240
|
+
|
|
241
|
+
expect(decision.decision).toBe("report")
|
|
242
|
+
expect(decision.reason).toBe("no escalation: CPU flatness is not proven (4000ms of CPU moved across 25000ms)")
|
|
243
|
+
})
|
|
244
|
+
|
|
245
|
+
test("a process that has not expired its retirement budget is never escalated", () => {
|
|
246
|
+
const previous = sample({atMs: 1000000, cpuMs: 5000, liveChildCount: 0, retirementTimedOut: false})
|
|
247
|
+
const current = sample({atMs: 1025000, cpuMs: 5000, liveChildCount: 0, retirementTimedOut: false})
|
|
248
|
+
const decision = escalate({current, previous})
|
|
249
|
+
|
|
250
|
+
expect(decision.decision).toBe("report")
|
|
251
|
+
expect(decision.reason).toBe("no escalation: drain is progressing (process worker is still inside its retirement budget)")
|
|
252
|
+
})
|
|
253
|
+
|
|
254
|
+
test("a generation that still owns a live connection is never escalated", () => {
|
|
255
|
+
const decision = escalate({connectionCount: 2, current: sample({liveChildCount: 0})})
|
|
256
|
+
|
|
257
|
+
expect(decision.decision).toBe("report")
|
|
258
|
+
expect(decision.reason).toBe("no escalation: drain is progressing (2 live connections still owned by the generation)")
|
|
259
|
+
})
|
|
260
|
+
|
|
261
|
+
test("a wedge with no measurable CPU is never escalated", () => {
|
|
262
|
+
const previous = sample({atMs: 1000000, cpuMs: undefined, liveChildCount: 0})
|
|
263
|
+
const current = sample({atMs: 1025000, cpuMs: undefined, liveChildCount: 0})
|
|
264
|
+
|
|
265
|
+
expect(classify({current, previous}).status).toBe("wedged")
|
|
266
|
+
|
|
267
|
+
const decision = escalate({current, previous})
|
|
268
|
+
|
|
269
|
+
expect(decision.decision).toBe("report")
|
|
270
|
+
expect(decision.reason).toBe("no escalation: CPU flatness is not proven (process CPU is not measurable on this host)")
|
|
271
|
+
})
|
|
272
|
+
|
|
273
|
+
test("a wedge seen in one observation is not escalated until a full window has passed", () => {
|
|
274
|
+
const current = sample({liveChildCount: 0})
|
|
275
|
+
const shortWindow = escalate({current: sample({atMs: 1001000, cpuMs: 5000, liveChildCount: 0}), previous: sample({atMs: 1000000, cpuMs: 5000, liveChildCount: 0})})
|
|
276
|
+
|
|
277
|
+
expect(classify({current}).status).toBe("wedged")
|
|
278
|
+
expect(escalate({current}).decision).toBe("report")
|
|
279
|
+
expect(escalate({current}).reason).toBe("no escalation: liveness window 0ms is below the 20000ms minimum")
|
|
280
|
+
expect(shortWindow.reason).toBe("no escalation: liveness window 1000ms is below the 20000ms minimum")
|
|
281
|
+
})
|
|
282
|
+
|
|
283
|
+
test("admission allows a handoff that stays within the bound and counts the incumbent", () => {
|
|
284
|
+
const admission = admitGenerationRelease({
|
|
285
|
+
candidateReleaseId: "v3",
|
|
286
|
+
draining: [{reason: "1 live connection still owned by the generation", releaseId: "v1", status: "progressing"}],
|
|
287
|
+
incumbentWillDrain: true,
|
|
288
|
+
maxConcurrentDrainingReleaseGroups: 2
|
|
289
|
+
})
|
|
290
|
+
|
|
291
|
+
expect(admission.allowed).toBe(true)
|
|
292
|
+
expect(admission.wouldBeDrainingCount).toBe(2)
|
|
293
|
+
expect(admission.reason).toBe(undefined)
|
|
294
|
+
})
|
|
295
|
+
|
|
296
|
+
test("admission refuses before activation and names every draining group with its classification", () => {
|
|
297
|
+
const admission = admitGenerationRelease({
|
|
298
|
+
candidateReleaseId: "20261115073717",
|
|
299
|
+
draining: [
|
|
300
|
+
{reason: "1 live process with 3 runner children, no live connection, every retirement budget expired, CPU flat across 25000ms", releaseId: "20261115063805", status: "wedged"},
|
|
301
|
+
{reason: "2 live connections still owned by the generation", releaseId: "20261114232900", status: "progressing"}
|
|
302
|
+
],
|
|
303
|
+
incumbentWillDrain: true,
|
|
304
|
+
maxConcurrentDrainingReleaseGroups: 2
|
|
305
|
+
})
|
|
306
|
+
|
|
307
|
+
expect(admission.allowed).toBe(false)
|
|
308
|
+
expect(admission.wouldBeDrainingCount).toBe(3)
|
|
309
|
+
expect(admission.reason).toContain("Deploy refused before activation: 2 release groups already draining")
|
|
310
|
+
expect(admission.reason).toContain("20261115063805 wedged")
|
|
311
|
+
expect(admission.reason).toContain("Activating 20261115073717 would make 3, above the releaseRetention.maxConcurrentDrainingReleaseGroups bound of 2")
|
|
312
|
+
})
|
|
313
|
+
|
|
314
|
+
test("a first deploy with no incumbent adds no draining group", () => {
|
|
315
|
+
const admission = admitGenerationRelease({candidateReleaseId: "v1", draining: [], incumbentWillDrain: false, maxConcurrentDrainingReleaseGroups: 1})
|
|
316
|
+
|
|
317
|
+
expect(admission.allowed).toBe(true)
|
|
318
|
+
expect(admission.wouldBeDrainingCount).toBe(0)
|
|
319
|
+
})
|
|
320
|
+
|
|
321
|
+
test("a bound of 0 disables the admission check", () => {
|
|
322
|
+
const draining = Array.from({length: 30}, (_, index) => ({reason: "no runner child remaining", releaseId: `v${index}`, status: /** @type {const} */ ("wedged")}))
|
|
323
|
+
const admission = admitGenerationRelease({candidateReleaseId: "v31", draining, incumbentWillDrain: true, maxConcurrentDrainingReleaseGroups: 0})
|
|
324
|
+
|
|
325
|
+
expect(admission.allowed).toBe(true)
|
|
326
|
+
expect(admission.wouldBeDrainingCount).toBe(31)
|
|
327
|
+
})
|
|
328
|
+
})
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
import http from "node:http"
|
|
4
|
+
import net from "node:net"
|
|
5
|
+
|
|
6
|
+
const port = Number(requiredEnv("ROLLBRIDGE_PORT"))
|
|
7
|
+
const bootMs = Number(process.env.ROLLBRIDGE_BOOT_MS || "0")
|
|
8
|
+
const dependencyPort = process.env.ROLLBRIDGE_DEPENDENCY_PORT
|
|
9
|
+
const releaseId = process.env.ROLLBRIDGE_RELEASE_ID || "unknown"
|
|
10
|
+
|
|
11
|
+
log(`starting (boot budget ${bootMs}ms)`)
|
|
12
|
+
|
|
13
|
+
if (dependencyPort) {
|
|
14
|
+
// The app performs its own dependency wait, exactly as the production web command does.
|
|
15
|
+
await waitForDependency(Number(dependencyPort))
|
|
16
|
+
log(`dependency ${dependencyPort} accepted a connection`)
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
await sleep(bootMs)
|
|
20
|
+
|
|
21
|
+
const server = http.createServer((request, response) => {
|
|
22
|
+
if (request.url === "/ping") {
|
|
23
|
+
response.writeHead(200, {"Content-Type": "application/json"})
|
|
24
|
+
response.end(JSON.stringify({message: "Pong", releaseId}))
|
|
25
|
+
return
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
response.writeHead(200, {"Content-Type": "text/plain; charset=utf-8"})
|
|
29
|
+
response.end(`${releaseId}\n`)
|
|
30
|
+
})
|
|
31
|
+
|
|
32
|
+
process.on("SIGTERM", () => {
|
|
33
|
+
server.close(() => process.exit(0))
|
|
34
|
+
setTimeout(() => process.exit(0), 10).unref()
|
|
35
|
+
})
|
|
36
|
+
|
|
37
|
+
server.listen(port, "127.0.0.1")
|
|
38
|
+
log(`listening on ${port}`)
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* @param {number} dependencyPortNumber - Port of the process this app depends on.
|
|
42
|
+
* @returns {Promise<void>} Resolves once the dependency accepts a connection.
|
|
43
|
+
*/
|
|
44
|
+
async function waitForDependency(dependencyPortNumber) {
|
|
45
|
+
const deadline = Date.now() + 10000
|
|
46
|
+
|
|
47
|
+
while (Date.now() < deadline) {
|
|
48
|
+
if (await tryConnect(dependencyPortNumber)) return
|
|
49
|
+
await sleep(50)
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
log(`dependency ${dependencyPortNumber} never accepted a connection`)
|
|
53
|
+
process.exit(21)
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* @param {number} dependencyPortNumber - Port to connect to.
|
|
58
|
+
* @returns {Promise<boolean>} True when the dependency accepted the connection.
|
|
59
|
+
*/
|
|
60
|
+
function tryConnect(dependencyPortNumber) {
|
|
61
|
+
return new Promise((resolve) => {
|
|
62
|
+
const socket = net.createConnection({host: "127.0.0.1", port: dependencyPortNumber})
|
|
63
|
+
let settled = false
|
|
64
|
+
|
|
65
|
+
/** @param {boolean} connected - Whether the connection succeeded. */
|
|
66
|
+
const finish = (connected) => {
|
|
67
|
+
if (settled) return
|
|
68
|
+
settled = true
|
|
69
|
+
socket.destroy()
|
|
70
|
+
resolve(connected)
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
socket.setTimeout(200)
|
|
74
|
+
socket.once("connect", () => finish(true))
|
|
75
|
+
socket.once("error", () => finish(false))
|
|
76
|
+
socket.once("timeout", () => finish(false))
|
|
77
|
+
})
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* @param {number} ms - Milliseconds.
|
|
82
|
+
* @returns {Promise<void>} Resolves after the delay.
|
|
83
|
+
*/
|
|
84
|
+
function sleep(ms) {
|
|
85
|
+
return new Promise((resolve) => setTimeout(resolve, ms))
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* @param {string} line - Output line, so the failure record can name the app's last output.
|
|
90
|
+
* @returns {void}
|
|
91
|
+
*/
|
|
92
|
+
function log(line) {
|
|
93
|
+
console.log(`boot: ${process.pid} ${line}`)
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* @param {string} key - Environment variable name.
|
|
98
|
+
* @returns {string} Environment variable value.
|
|
99
|
+
*/
|
|
100
|
+
function requiredEnv(key) {
|
|
101
|
+
const value = process.env[key]
|
|
102
|
+
|
|
103
|
+
if (!value) throw new Error(`${key} is required`)
|
|
104
|
+
|
|
105
|
+
return value
|
|
106
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
import http from "node:http"
|
|
4
|
+
import fs from "node:fs"
|
|
5
|
+
|
|
6
|
+
const port = Number(requiredEnv("ROLLBRIDGE_PORT"))
|
|
7
|
+
const gateArg = process.argv.find((arg) => arg.startsWith("--gate="))
|
|
8
|
+
|
|
9
|
+
if (!gateArg) throw new Error("--gate=<path> is required")
|
|
10
|
+
|
|
11
|
+
const gatePath = gateArg.slice("--gate=".length)
|
|
12
|
+
const releaseId = process.env.ROLLBRIDGE_RELEASE_ID || "unknown"
|
|
13
|
+
// v3 stays unhealthy until the gate file appears, so its deploy fails at health.
|
|
14
|
+
const unhealthy = releaseId === "v3" && !fs.existsSync(gatePath)
|
|
15
|
+
|
|
16
|
+
const server = http.createServer((request, response) => {
|
|
17
|
+
if (unhealthy) {
|
|
18
|
+
response.writeHead(500)
|
|
19
|
+
response.end("bad")
|
|
20
|
+
return
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
if (request.url === "/ping") {
|
|
24
|
+
response.writeHead(200, {"Content-Type": "application/json"})
|
|
25
|
+
response.end(JSON.stringify({message: "Pong", releaseId}))
|
|
26
|
+
return
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
response.writeHead(200, {"Content-Type": "text/plain; charset=utf-8"})
|
|
30
|
+
response.end(`${releaseId}\n`)
|
|
31
|
+
})
|
|
32
|
+
|
|
33
|
+
process.on("SIGTERM", () => {
|
|
34
|
+
server.close(() => process.exit(0))
|
|
35
|
+
setTimeout(() => process.exit(0), 10).unref()
|
|
36
|
+
})
|
|
37
|
+
|
|
38
|
+
server.listen(port, "127.0.0.1")
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* @param {string} key - Environment variable name.
|
|
42
|
+
* @returns {string} Environment variable value.
|
|
43
|
+
*/
|
|
44
|
+
function requiredEnv(key) {
|
|
45
|
+
const value = process.env[key]
|
|
46
|
+
|
|
47
|
+
if (!value) throw new Error(`${key} is required`)
|
|
48
|
+
|
|
49
|
+
return value
|
|
50
|
+
}
|
|
@@ -7,19 +7,39 @@ import os from "node:os"
|
|
|
7
7
|
import path from "node:path"
|
|
8
8
|
import {describe, expect, test} from "@velocious/testing"
|
|
9
9
|
import {fileURLToPath} from "node:url"
|
|
10
|
-
import
|
|
10
|
+
import {createRequire} from "node:module"
|
|
11
|
+
import GuardianClient, {usableSocketPath} from "../src/guardian-client.js"
|
|
11
12
|
import {waitForProcessExit} from "./support/process.js"
|
|
12
13
|
|
|
13
14
|
describe("guardian-client", () => {
|
|
14
15
|
|
|
16
|
+
const commonJsRequire = createRequire(import.meta.url)
|
|
17
|
+
const packageVersion = /** @type {{version: string}} */ (commonJsRequire("../package.json")).version
|
|
15
18
|
const legacyGuardianPath = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures", "pre-split3-process-guardian.js")
|
|
16
19
|
const recoveryOwnerPath = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures", "guardian-recovery-owner.js")
|
|
17
20
|
|
|
21
|
+
test("guardian maps an overlong configured socket path to a stable short path", async () => {
|
|
22
|
+
const root = await fs.mkdtemp(path.join(os.tmpdir(), `rollbridge-${"x".repeat(120)}-`))
|
|
23
|
+
const configuredPath = path.join(root, "guardian.sock")
|
|
24
|
+
const client = new GuardianClient({socketPath: configuredPath, token: "long-path-capability"})
|
|
25
|
+
|
|
26
|
+
try {
|
|
27
|
+
expect(Buffer.byteLength(configuredPath)).toBeGreaterThan(107)
|
|
28
|
+
expect(Buffer.byteLength(client.socketPath)).toBeLessThanOrEqual(107)
|
|
29
|
+
expect(client.socketPath).not.toBe(configuredPath)
|
|
30
|
+
await client.launch()
|
|
31
|
+
expect(await client.capabilities()).toEqual({daemonRecovery: 1, generationReactivation: 1, guardianVersion: packageVersion})
|
|
32
|
+
} finally {
|
|
33
|
+
await cleanupGuardian({client, root})
|
|
34
|
+
}
|
|
35
|
+
})
|
|
36
|
+
|
|
18
37
|
test("guardian bootstrap capability is absent from process argv", async () => {
|
|
19
38
|
const fixture = await createGuardian()
|
|
39
|
+
const packageVersion = JSON.parse(await fs.readFile(path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "package.json"), "utf8")).version
|
|
20
40
|
|
|
21
41
|
try {
|
|
22
|
-
expect(await fixture.client.capabilities()).toEqual({daemonRecovery: 1, generationReactivation: 1})
|
|
42
|
+
expect(await fixture.client.capabilities()).toEqual({daemonRecovery: 1, generationReactivation: 1, guardianVersion: packageVersion})
|
|
23
43
|
const commandLine = await fs.readFile(`/proc/${fixture.client.pid}/cmdline`, "utf8")
|
|
24
44
|
const environment = await fs.readFile(`/proc/${fixture.client.pid}/environ`, "utf8")
|
|
25
45
|
const status = await fs.readFile(`/proc/${fixture.client.pid}/status`, "utf8")
|
|
@@ -58,6 +78,100 @@ test("guardian inventory removes only an exact owned provenance", async () => {
|
|
|
58
78
|
}
|
|
59
79
|
})
|
|
60
80
|
|
|
81
|
+
test("a refused owner mutation names the held mutation, its process, its age and its blocker", async () => {
|
|
82
|
+
const fixture = await createGuardian()
|
|
83
|
+
|
|
84
|
+
try {
|
|
85
|
+
const mutationId = await fixture.client.beginOwnerMutation("restart", {ownerPid: process.pid, waitingFor: "the managed process to stop and start again"})
|
|
86
|
+
await expect(fixture.client.beginOwnerMutation("deploy")).rejects.toThrow(/^Another owner mutation is already in progress: owner mutation "restart"; mutation [0-9a-f-]{36}; held by process \d+; since 20\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\.\d\d\dZ; \d+s elapsed; waiting for the managed process to stop and start again; refused deploy$/)
|
|
87
|
+
await fixture.client.reportOwnerMutationWait(mutationId, "an idle forked runner that never exits")
|
|
88
|
+
await expect(fixture.client.beginOwnerMutation("deploy")).rejects.toThrow(/waiting for an idle forked runner that never exits/)
|
|
89
|
+
await fixture.client.endOwnerMutation(mutationId)
|
|
90
|
+
expect(typeof await fixture.client.beginOwnerMutation("deploy")).toBe("string")
|
|
91
|
+
} finally {
|
|
92
|
+
await cleanupGuardian(fixture)
|
|
93
|
+
}
|
|
94
|
+
})
|
|
95
|
+
|
|
96
|
+
test("an owner mutation that outruns its lease is released and the holder reports the expiry", async () => {
|
|
97
|
+
const fixture = await createGuardian()
|
|
98
|
+
|
|
99
|
+
try {
|
|
100
|
+
const wedged = await fixture.client.beginOwnerMutation("restart", {mutationTimeoutMs: 200, ownerPid: process.pid, waitingFor: "a drain that never returns"})
|
|
101
|
+
|
|
102
|
+
await new Promise((resolve) => setTimeout(resolve, 600))
|
|
103
|
+
const successor = await fixture.client.beginOwnerMutation("deploy", {ownerPid: process.pid, waitingFor: "candidate activation"})
|
|
104
|
+
|
|
105
|
+
expect(typeof successor).toBe("string")
|
|
106
|
+
await expect(fixture.client.endOwnerMutation(wedged)).rejects.toThrow(/^Owner mutation restart exceeded its 200 ms lease and was released after \d+ ms; the mutation failed and its lease is gone$/)
|
|
107
|
+
await fixture.client.endOwnerMutation(successor)
|
|
108
|
+
} finally {
|
|
109
|
+
await cleanupGuardian(fixture)
|
|
110
|
+
}
|
|
111
|
+
})
|
|
112
|
+
|
|
113
|
+
test("a dead owner-mutation holder is reclaimed and the next mutation is admitted", async () => {
|
|
114
|
+
const fixture = await createGuardian()
|
|
115
|
+
const holder = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"])
|
|
116
|
+
|
|
117
|
+
try {
|
|
118
|
+
const reclaimed = fixture.client.waitForEvent("owner-mutation-reclaimed")
|
|
119
|
+
await fixture.client.beginOwnerMutation("restart", {ownerPid: holder.pid, waitingFor: "a process that will die"})
|
|
120
|
+
if (!holder.pid) throw new Error("Missing mutation holder PID")
|
|
121
|
+
holder.kill("SIGKILL")
|
|
122
|
+
await once(holder, "exit")
|
|
123
|
+
|
|
124
|
+
const successor = await fixture.client.beginOwnerMutation("deploy", {ownerPid: process.pid, waitingFor: "candidate activation"})
|
|
125
|
+
const event = await reclaimed
|
|
126
|
+
|
|
127
|
+
expect(typeof successor).toBe("string")
|
|
128
|
+
expect(event).toEqual(expect.objectContaining({code: "OWNER_MUTATION_RECLAIMED", event: "owner-mutation-reclaimed", operation: "restart", pid: holder.pid, reason: "owner mutation holder process is no longer alive"}))
|
|
129
|
+
expect((await fixture.client.guardianStatus()).version).toBe(packageVersion)
|
|
130
|
+
await fixture.client.endOwnerMutation(successor)
|
|
131
|
+
} finally {
|
|
132
|
+
if (holder.exitCode === null && holder.signalCode === null) holder.kill("SIGKILL")
|
|
133
|
+
await cleanupGuardian(fixture)
|
|
134
|
+
}
|
|
135
|
+
})
|
|
136
|
+
|
|
137
|
+
test("guardian lets the owner re-register a key whose registered child is gone, and still fences a live one", async () => {
|
|
138
|
+
const fixture = await createGuardian()
|
|
139
|
+
const liveInstance = fixture.client.process("live-generation", definition("live-generation"))
|
|
140
|
+
const deadInstance = fixture.client.process("dead-generation", definition("dead-generation"))
|
|
141
|
+
|
|
142
|
+
try {
|
|
143
|
+
await liveInstance.start()
|
|
144
|
+
await deadInstance.start()
|
|
145
|
+
const deadPid = (await deadInstance.status()).pid
|
|
146
|
+
|
|
147
|
+
if (!deadPid) throw new Error("Missing required fixture: dead-generation pid")
|
|
148
|
+
await deadInstance.stop()
|
|
149
|
+
await waitForProcessExit(deadPid)
|
|
150
|
+
const deadStatus = await deadInstance.status()
|
|
151
|
+
|
|
152
|
+
expect(deadStatus.state).toBe("stopped")
|
|
153
|
+
expect(deadStatus.pid).toBeUndefined()
|
|
154
|
+
|
|
155
|
+
// The wedge this guards against: a later deploy reconstructs the same key from a
|
|
156
|
+
// different release definition. The old child is conclusively gone, so the old
|
|
157
|
+
// provenance must not fence the new registration.
|
|
158
|
+
const nextInstance = fixture.client.process("dead-generation", {...definition("dead-generation"), env: {REPLACED_BY_LATER_DEPLOY: "1"}})
|
|
159
|
+
const reRegistered = nextInstance.status()
|
|
160
|
+
|
|
161
|
+
expect(reRegistered.state).toBe("stopped")
|
|
162
|
+
await nextInstance.start()
|
|
163
|
+
expect((await nextInstance.status()).state).toBe("running")
|
|
164
|
+
expect((await fixture.client.inventory()).some((entry) => entry.key === "dead-generation" && entry.provenance === nextInstance.provenance)).toBe(true)
|
|
165
|
+
|
|
166
|
+
// A live process with a different provenance stays fail-closed.
|
|
167
|
+
const hostileInstance = fixture.client.process("live-generation", {...definition("live-generation"), env: {HOSTILE: "1"}})
|
|
168
|
+
|
|
169
|
+
await expect(hostileInstance.start()).rejects.toThrow(/provenance mismatch/)
|
|
170
|
+
} finally {
|
|
171
|
+
await cleanupGuardian(fixture)
|
|
172
|
+
}
|
|
173
|
+
})
|
|
174
|
+
|
|
61
175
|
test("guardian runs a strict activation lifecycle command for the exact registered process", async () => {
|
|
62
176
|
const fixture = await createGuardian()
|
|
63
177
|
const activationPath = path.join(fixture.root, "activated")
|
|
@@ -842,7 +956,7 @@ test("replacement staging rejects owner state published after prepare", async ()
|
|
|
842
956
|
const prepared = await candidate.prepareOwnerReplacement(authority, nextAuthority)
|
|
843
957
|
|
|
844
958
|
await fixture.client.publishOwnerState({authority, snapshot: {activeReleaseId: "v2"}})
|
|
845
|
-
await expect(candidate.stageOwnerReplacement(prepared.replacementId, {authority: nextAuthority, snapshot: {activeReleaseId: "v1"}})).rejects.toThrow(/owner state changed after prepare/i)
|
|
959
|
+
await expect(candidate.stageOwnerReplacement(prepared.replacementId, {authority: nextAuthority, snapshot: {activeReleaseId: "v1"}})).rejects.toThrow(/owner state changed after prepare \(prepared revision 1, current revision 2\)/i)
|
|
846
960
|
await candidate.abortOwnerReplacement(prepared.replacementId)
|
|
847
961
|
const fresh = await candidate.prepareOwnerReplacement(authority, nextAuthority)
|
|
848
962
|
|
|
@@ -1678,7 +1792,7 @@ test("a legacy bridge remains discoverable when its candidate disconnects after
|
|
|
1678
1792
|
let replacement
|
|
1679
1793
|
|
|
1680
1794
|
try {
|
|
1681
|
-
const identity = {socketPath: path.join(fixture.root, "guardian-v2.sock"), token: "candidate-guardian-capability"}
|
|
1795
|
+
const identity = {socketPath: usableSocketPath(path.join(fixture.root, "guardian-v2.sock")), token: "candidate-guardian-capability"}
|
|
1682
1796
|
|
|
1683
1797
|
upgraded = await fixture.client.upgradeLegacyGuardian({ownerState, ...identity})
|
|
1684
1798
|
const prepared = await upgraded.prepareOwnerReplacement(authority, nextAuthority)
|
|
@@ -1725,7 +1839,7 @@ async function createGuardian() {
|
|
|
1725
1839
|
async function createLegacyGuardian() {
|
|
1726
1840
|
const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-legacy-guardian-"))
|
|
1727
1841
|
const token = "legacy-guardian-capability"
|
|
1728
|
-
const socketPath = path.join(root, "guardian.sock")
|
|
1842
|
+
const socketPath = usableSocketPath(path.join(root, "guardian.sock"))
|
|
1729
1843
|
const child = spawn(process.execPath, [legacyGuardianPath, socketPath], {detached: true, stdio: ["ignore", "ignore", "ignore", "ipc"]})
|
|
1730
1844
|
|
|
1731
1845
|
await new Promise((resolve, reject) => {
|