rollbridge 0.1.70 → 0.1.71
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -63
- package/TODO.md +0 -13
- package/docs/cli.md +3 -128
- package/docs/config.md +6 -177
- package/docs/troubleshooting.md +6 -97
- package/docs/velocious.md +1 -1
- package/docs/workers.md +0 -11
- package/examples/tensorbuzz.com.js +2 -3
- package/package.json +1 -1
- package/src/cli.js +2 -68
- package/src/config.js +8 -347
- package/src/daemon.js +65 -387
- package/src/doctor.js +0 -74
- package/src/guardian-client.js +8 -67
- package/src/health.js +8 -99
- package/src/managed-process.js +16 -197
- package/src/process-guardian.js +22 -212
- package/src/process-memory.js +2 -62
- package/src/release-group.js +37 -435
- package/tensorbuzz.yml +1 -20
- package/test/completion.test.js +2 -5
- package/test/config-validation.test.js +5 -135
- package/test/guardian-client.test.js +5 -119
- package/test/health.test.js +1 -74
- package/test/managed-process.test.js +1 -260
- package/test/owner-recovery.test.js +21 -125
- package/test/owner-replacement.test.js +14 -175
- package/test/process-memory.test.js +1 -47
- package/test/release-group.test.js +2 -408
- package/test/rollbridge.test.js +72 -737
- package/test/shutdown-completion.test.js +0 -52
- package/test/support/process.js +1 -3
- package/changelog.d/20261025-guardian-re-register-dead-generation.md +0 -13
- package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +0 -1
- package/changelog.d/20261104-compose-service-units.md +0 -21
- package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +0 -20
- package/changelog.d/20261106-release-dependency-boot-budgets.md +0 -32
- package/changelog.d/20261109-path-filtered-builds.md +0 -11
- package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +0 -26
- package/changelog.d/20261111-contention-sensitive-daemon-specs.md +0 -27
- package/changelog.d/20261113-bounded-nonblocking-retirement.md +0 -5
- package/changelog.d/20261115-bound-draining-release-groups.md +0 -62
- package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +0 -27
- package/changelog.d/20261115-wedged-generation-escalation.md +0 -44
- package/changelog.d/20261116-bounded-explicit-stop-retirement.md +0 -30
- package/changelog.d/20261116-config-json-canonical-shape.md +0 -11
- package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +0 -29
- package/changelog.d/20261116-guardian-version-handoff.md +0 -37
- package/changelog.d/20261116-reclaim-dead-owner-mutation.md +0 -7
- package/changelog.d/20261117-activate-before-incumbent-retirement.md +0 -1
- package/changelog.d/20261117-activation-path-drain-deadline.md +0 -35
- package/changelog.d/20261117-stop-companion-after-retired-dependency.md +0 -7
- package/docs/compose-services.md +0 -222
- package/src/compose-unit.js +0 -725
- package/src/drain-policy.js +0 -252
- package/src/process-order.js +0 -26
- package/test/compose-config.test.js +0 -160
- package/test/compose-docker.test.js +0 -297
- package/test/compose-unit.test.js +0 -415
- package/test/drain-policy.test.js +0 -328
- package/test/fixtures/deferred-boot-app.js +0 -106
- package/test/fixtures/gated-app.js +0 -50
- package/test/tensorbuzz-config.test.js +0 -228
|
@@ -1,328 +0,0 @@
|
|
|
1
|
-
// @ts-check
|
|
2
|
-
|
|
3
|
-
import {describe, expect, test} from "@velocious/testing"
|
|
4
|
-
import {admitGenerationRelease, classifyDrainProgress, decideWedgedEscalation} from "../src/drain-policy.js"
|
|
5
|
-
|
|
6
|
-
describe("drain-policy", () => {
|
|
7
|
-
|
|
8
|
-
const WINDOW_MS = 20000
|
|
9
|
-
|
|
10
|
-
/**
|
|
11
|
-
* @param {object} [options] - Observation shape.
|
|
12
|
-
* @param {number} [options.atMs] - Observation time.
|
|
13
|
-
* @param {number | undefined} [options.cpuMs] - Cumulative CPU of the group.
|
|
14
|
-
* @param {number | undefined} [options.heldProcessGroupPgid] - Group still alive after its tracked process exited.
|
|
15
|
-
* @param {number} [options.liveChildCount] - Runner children other than the group leader.
|
|
16
|
-
* @param {number | undefined} [options.pid] - Live pid, or undefined when the process is gone.
|
|
17
|
-
* @param {boolean} [options.retirementTimedOut] - Whether the supervisor already gave up on it.
|
|
18
|
-
* @returns {import("../src/drain-policy.js").DrainLivenessSample} One liveness observation.
|
|
19
|
-
*/
|
|
20
|
-
function sample(options = {}) {
|
|
21
|
-
return {
|
|
22
|
-
atMs: options.atMs ?? 1000000,
|
|
23
|
-
processes: [{
|
|
24
|
-
cpuMs: "cpuMs" in options ? options.cpuMs : 5000,
|
|
25
|
-
heldProcessGroupPgid: options.heldProcessGroupPgid,
|
|
26
|
-
id: "worker",
|
|
27
|
-
liveChildCount: options.liveChildCount ?? 1,
|
|
28
|
-
pid: "pid" in options ? options.pid : 4242,
|
|
29
|
-
retirementTimedOut: options.retirementTimedOut ?? true
|
|
30
|
-
}]
|
|
31
|
-
}
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
/**
|
|
35
|
-
* @param {object} [options] - Classification inputs.
|
|
36
|
-
* @param {number} [options.connectionCount] - Live connections owned by the generation.
|
|
37
|
-
* @param {import("../src/drain-policy.js").DrainLivenessSample} [options.current] - Newest observation.
|
|
38
|
-
* @param {number} [options.drainBudgetMs] - Configured drain budget.
|
|
39
|
-
* @param {number} [options.drainElapsedMs] - Time since the drain started.
|
|
40
|
-
* @param {import("../src/drain-policy.js").DrainLivenessSample | undefined} [options.previous] - Older observation.
|
|
41
|
-
* @returns {import("../src/drain-policy.js").DrainClassification} Classification.
|
|
42
|
-
*/
|
|
43
|
-
function classify(options = {}) {
|
|
44
|
-
return classifyDrainProgress({
|
|
45
|
-
connectionCount: options.connectionCount ?? 0,
|
|
46
|
-
currentSample: options.current ?? sample(),
|
|
47
|
-
drainBudgetMs: options.drainBudgetMs ?? 0,
|
|
48
|
-
drainElapsedMs: options.drainElapsedMs ?? 0,
|
|
49
|
-
minSampleWindowMs: WINDOW_MS,
|
|
50
|
-
previousSample: options.previous
|
|
51
|
-
})
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
/**
|
|
55
|
-
* @param {object} [options] - Decision inputs.
|
|
56
|
-
* @param {number} [options.connectionCount] - Live connections owned by the generation.
|
|
57
|
-
* @param {import("../src/drain-policy.js").DrainLivenessSample} [options.current] - Newest observation.
|
|
58
|
-
* @param {import("../src/drain-policy.js").DrainLivenessSample | undefined} [options.previous] - Older observation.
|
|
59
|
-
* @returns {import("../src/drain-policy.js").EscalationDecision} Escalation decision for the same pair the classifier saw.
|
|
60
|
-
*/
|
|
61
|
-
function escalate(options = {}) {
|
|
62
|
-
const currentSample = options.current ?? sample()
|
|
63
|
-
|
|
64
|
-
return decideWedgedEscalation({
|
|
65
|
-
classification: classify({connectionCount: options.connectionCount, current: currentSample, previous: options.previous}),
|
|
66
|
-
currentSample,
|
|
67
|
-
minSampleWindowMs: WINDOW_MS
|
|
68
|
-
})
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
test("a generation that still owns connections is draining, never wedged", () => {
|
|
72
|
-
const result = classify({connectionCount: 3, current: sample({liveChildCount: 0})})
|
|
73
|
-
|
|
74
|
-
expect(result.status).toBe("progressing")
|
|
75
|
-
expect(result.reason).toBe("3 live connections still owned by the generation")
|
|
76
|
-
})
|
|
77
|
-
|
|
78
|
-
test("a live connection past the configured drain budget is reported past the deadline, not progressing", () => {
|
|
79
|
-
const result = classify({connectionCount: 1, current: sample({liveChildCount: 0}), drainBudgetMs: 60000, drainElapsedMs: 61000})
|
|
80
|
-
|
|
81
|
-
expect(result.status).toBe("timedOut")
|
|
82
|
-
expect(result.reason).toBe("drain budget 60000ms exceeded by 1000ms with 1 live connection still owned by the generation")
|
|
83
|
-
})
|
|
84
|
-
|
|
85
|
-
test("a live connection inside the configured drain budget is still a legitimate drain", () => {
|
|
86
|
-
const result = classify({connectionCount: 1, current: sample({liveChildCount: 0}), drainBudgetMs: 60000, drainElapsedMs: 30000})
|
|
87
|
-
|
|
88
|
-
expect(result.status).toBe("progressing")
|
|
89
|
-
expect(result.reason).toBe("1 live connection still owned by the generation")
|
|
90
|
-
})
|
|
91
|
-
|
|
92
|
-
test("the drain deadline names the deadline and never escalates", () => {
|
|
93
|
-
const currentSample = sample({liveChildCount: 0})
|
|
94
|
-
const classification = classify({connectionCount: 1, current: currentSample, drainBudgetMs: 60000, drainElapsedMs: 61000})
|
|
95
|
-
const decision = decideWedgedEscalation({classification, currentSample, minSampleWindowMs: WINDOW_MS})
|
|
96
|
-
|
|
97
|
-
expect(decision.decision).toBe("report")
|
|
98
|
-
expect(decision.reason).toBe("no escalation: drain is timedOut (drain budget 60000ms exceeded by 1000ms with 1 live connection still owned by the generation)")
|
|
99
|
-
expect(decision.targets).toEqual([])
|
|
100
|
-
})
|
|
101
|
-
|
|
102
|
-
test("a process still inside its retirement budget is draining", () => {
|
|
103
|
-
const result = classify({current: sample({liveChildCount: 0, retirementTimedOut: false})})
|
|
104
|
-
|
|
105
|
-
expect(result.status).toBe("progressing")
|
|
106
|
-
expect(result.reason).toBe("process worker is still inside its retirement budget")
|
|
107
|
-
})
|
|
108
|
-
|
|
109
|
-
test("a live process past its retirement budget with no runner child left is wedged", () => {
|
|
110
|
-
const result = classify({current: sample({liveChildCount: 0})})
|
|
111
|
-
|
|
112
|
-
expect(result.status).toBe("wedged")
|
|
113
|
-
expect(result.reason).toBe("1 live process past its retirement budget with no runner child remaining")
|
|
114
|
-
})
|
|
115
|
-
|
|
116
|
-
test("flat CPU across a sufficient window is wedged, and the window is reported", () => {
|
|
117
|
-
const previous = sample({atMs: 1000000, cpuMs: 5000})
|
|
118
|
-
const current = sample({atMs: 1025000, cpuMs: 5000, liveChildCount: 3})
|
|
119
|
-
const result = classify({current, previous})
|
|
120
|
-
|
|
121
|
-
expect(result.status).toBe("wedged")
|
|
122
|
-
expect(result.sampleWindowMs).toBe(25000)
|
|
123
|
-
expect(result.cpuDeltaMs).toBe(0)
|
|
124
|
-
expect(result.reason).toBe("1 live process with 3 runner children, no live connection, every retirement budget expired, CPU flat across 25000ms")
|
|
125
|
-
})
|
|
126
|
-
|
|
127
|
-
test("CPU that advanced across the window is a progressing drain", () => {
|
|
128
|
-
const previous = sample({atMs: 1000000, cpuMs: 5000})
|
|
129
|
-
const current = sample({atMs: 1025000, cpuMs: 9000})
|
|
130
|
-
const result = classify({current, previous})
|
|
131
|
-
|
|
132
|
-
expect(result.status).toBe("progressing")
|
|
133
|
-
expect(result.cpuDeltaMs).toBe(4000)
|
|
134
|
-
expect(result.reason).toBe("CPU advanced 4000ms across 25000ms")
|
|
135
|
-
})
|
|
136
|
-
|
|
137
|
-
test("CPU leaving the group with an exited member is a progressing drain", () => {
|
|
138
|
-
const previous = sample({atMs: 1000000, cpuMs: 9000})
|
|
139
|
-
const current = sample({atMs: 1025000, cpuMs: 5000})
|
|
140
|
-
const result = classify({current, previous})
|
|
141
|
-
|
|
142
|
-
expect(result.status).toBe("progressing")
|
|
143
|
-
expect(result.cpuDeltaMs).toBe(-4000)
|
|
144
|
-
expect(result.reason).toBe("the group changed (1 live process, -4000ms of consumed CPU left the group) across 25000ms")
|
|
145
|
-
})
|
|
146
|
-
|
|
147
|
-
test("the first observation cannot claim flatness", () => {
|
|
148
|
-
const result = classify({current: sample()})
|
|
149
|
-
|
|
150
|
-
expect(result.status).toBe("unproven")
|
|
151
|
-
expect(result.sampleWindowMs).toBe(0)
|
|
152
|
-
expect(result.reason).toBe("first liveness observation; a 20000ms window is required before CPU flatness can be claimed")
|
|
153
|
-
})
|
|
154
|
-
|
|
155
|
-
test("a window shorter than the minimum cannot claim flatness", () => {
|
|
156
|
-
const previous = sample({atMs: 1000000})
|
|
157
|
-
const current = sample({atMs: 1010000})
|
|
158
|
-
const result = classify({current, previous})
|
|
159
|
-
|
|
160
|
-
expect(result.status).toBe("unproven")
|
|
161
|
-
expect(result.sampleWindowMs).toBe(10000)
|
|
162
|
-
expect(result.reason).toBe("liveness window 10000ms is below the 20000ms minimum")
|
|
163
|
-
})
|
|
164
|
-
|
|
165
|
-
test("an unmeasurable CPU counter is unproven, never wedged", () => {
|
|
166
|
-
const previous = sample({atMs: 1000000, cpuMs: undefined})
|
|
167
|
-
const current = sample({atMs: 1025000, cpuMs: undefined})
|
|
168
|
-
const result = classify({current, previous})
|
|
169
|
-
|
|
170
|
-
expect(result.status).toBe("unproven")
|
|
171
|
-
expect(result.cpuDeltaMs).toBe(undefined)
|
|
172
|
-
expect(result.reason).toBe("process CPU is not measurable on this host")
|
|
173
|
-
})
|
|
174
|
-
|
|
175
|
-
test("a generation with no live process left is not a wedge", () => {
|
|
176
|
-
const result = classify({current: sample({pid: undefined})})
|
|
177
|
-
|
|
178
|
-
expect(result.status).toBe("progressing")
|
|
179
|
-
expect(result.reason).toBe("no live process remains in the generation")
|
|
180
|
-
})
|
|
181
|
-
|
|
182
|
-
test("a process group that outlived its tracked process is still a live process of the generation", () => {
|
|
183
|
-
const result = classify({current: sample({heldProcessGroupPgid: 4242, liveChildCount: 0, pid: undefined})})
|
|
184
|
-
|
|
185
|
-
expect(result.status).toBe("wedged")
|
|
186
|
-
expect(result.reason).toBe("1 live process past its retirement budget with no runner child remaining; 1 tracked process already exited and left worker pgid 4242 alive")
|
|
187
|
-
})
|
|
188
|
-
|
|
189
|
-
test("a held process group that still consumes CPU across the window is a progressing drain", () => {
|
|
190
|
-
const previous = sample({atMs: 1000000, cpuMs: 5000, heldProcessGroupPgid: 4242, liveChildCount: 2, pid: undefined})
|
|
191
|
-
const current = sample({atMs: 1025000, cpuMs: 9000, heldProcessGroupPgid: 4242, liveChildCount: 2, pid: undefined})
|
|
192
|
-
const result = classify({current, previous})
|
|
193
|
-
|
|
194
|
-
expect(result.status).toBe("progressing")
|
|
195
|
-
expect(result.cpuDeltaMs).toBe(4000)
|
|
196
|
-
expect(result.reason).toBe("CPU advanced 4000ms across 25000ms")
|
|
197
|
-
})
|
|
198
|
-
|
|
199
|
-
test("a proven wedge with no runner child left is escalated, naming the process group it terminates", () => {
|
|
200
|
-
const previous = sample({atMs: 1000000, cpuMs: 5000, liveChildCount: 3})
|
|
201
|
-
const current = sample({atMs: 1025000, cpuMs: 5000, liveChildCount: 0})
|
|
202
|
-
const decision = escalate({current, previous})
|
|
203
|
-
|
|
204
|
-
expect(decision.decision).toBe("escalate")
|
|
205
|
-
expect(decision.killSafe).toBe(true)
|
|
206
|
-
expect(decision.targets).toEqual([{pgid: 4242, processId: "worker"}])
|
|
207
|
-
expect(decision.reason).toContain("CPU flat across 25000ms")
|
|
208
|
-
})
|
|
209
|
-
|
|
210
|
-
test("the escalation targets the group that outlived its tracked process", () => {
|
|
211
|
-
const previous = sample({atMs: 1000000, cpuMs: 5000, heldProcessGroupPgid: 4242, liveChildCount: 2, pid: undefined})
|
|
212
|
-
const current = sample({atMs: 1025000, cpuMs: 5000, heldProcessGroupPgid: 4242, liveChildCount: 0, pid: undefined})
|
|
213
|
-
const decision = escalate({current, previous})
|
|
214
|
-
|
|
215
|
-
expect(decision.decision).toBe("escalate")
|
|
216
|
-
expect(decision.targets).toEqual([{pgid: 4242, processId: "worker"}])
|
|
217
|
-
})
|
|
218
|
-
|
|
219
|
-
test("a wedge that still holds a live runner child is never escalated", () => {
|
|
220
|
-
const previous = sample({atMs: 1000000, cpuMs: 5000})
|
|
221
|
-
const current = sample({atMs: 1025000, cpuMs: 5000, liveChildCount: 3})
|
|
222
|
-
|
|
223
|
-
expect(classify({current, previous}).status).toBe("wedged")
|
|
224
|
-
|
|
225
|
-
const decision = escalate({current, previous})
|
|
226
|
-
|
|
227
|
-
expect(decision.decision).toBe("report")
|
|
228
|
-
expect(decision.killSafe).toBe(false)
|
|
229
|
-
expect(decision.targets).toEqual([])
|
|
230
|
-
expect(decision.reason).toBe("no escalation: 3 runner children are still live in the generation; SIGKILL would abort in-flight work")
|
|
231
|
-
})
|
|
232
|
-
|
|
233
|
-
test("a live process whose CPU is still moving is never escalated", () => {
|
|
234
|
-
const previous = sample({atMs: 1000000, cpuMs: 5000})
|
|
235
|
-
const current = sample({atMs: 1025000, cpuMs: 9000, liveChildCount: 0})
|
|
236
|
-
|
|
237
|
-
expect(classify({current, previous}).status).toBe("wedged")
|
|
238
|
-
|
|
239
|
-
const decision = escalate({current, previous})
|
|
240
|
-
|
|
241
|
-
expect(decision.decision).toBe("report")
|
|
242
|
-
expect(decision.reason).toBe("no escalation: CPU flatness is not proven (4000ms of CPU moved across 25000ms)")
|
|
243
|
-
})
|
|
244
|
-
|
|
245
|
-
test("a process that has not expired its retirement budget is never escalated", () => {
|
|
246
|
-
const previous = sample({atMs: 1000000, cpuMs: 5000, liveChildCount: 0, retirementTimedOut: false})
|
|
247
|
-
const current = sample({atMs: 1025000, cpuMs: 5000, liveChildCount: 0, retirementTimedOut: false})
|
|
248
|
-
const decision = escalate({current, previous})
|
|
249
|
-
|
|
250
|
-
expect(decision.decision).toBe("report")
|
|
251
|
-
expect(decision.reason).toBe("no escalation: drain is progressing (process worker is still inside its retirement budget)")
|
|
252
|
-
})
|
|
253
|
-
|
|
254
|
-
test("a generation that still owns a live connection is never escalated", () => {
|
|
255
|
-
const decision = escalate({connectionCount: 2, current: sample({liveChildCount: 0})})
|
|
256
|
-
|
|
257
|
-
expect(decision.decision).toBe("report")
|
|
258
|
-
expect(decision.reason).toBe("no escalation: drain is progressing (2 live connections still owned by the generation)")
|
|
259
|
-
})
|
|
260
|
-
|
|
261
|
-
test("a wedge with no measurable CPU is never escalated", () => {
|
|
262
|
-
const previous = sample({atMs: 1000000, cpuMs: undefined, liveChildCount: 0})
|
|
263
|
-
const current = sample({atMs: 1025000, cpuMs: undefined, liveChildCount: 0})
|
|
264
|
-
|
|
265
|
-
expect(classify({current, previous}).status).toBe("wedged")
|
|
266
|
-
|
|
267
|
-
const decision = escalate({current, previous})
|
|
268
|
-
|
|
269
|
-
expect(decision.decision).toBe("report")
|
|
270
|
-
expect(decision.reason).toBe("no escalation: CPU flatness is not proven (process CPU is not measurable on this host)")
|
|
271
|
-
})
|
|
272
|
-
|
|
273
|
-
test("a wedge seen in one observation is not escalated until a full window has passed", () => {
|
|
274
|
-
const current = sample({liveChildCount: 0})
|
|
275
|
-
const shortWindow = escalate({current: sample({atMs: 1001000, cpuMs: 5000, liveChildCount: 0}), previous: sample({atMs: 1000000, cpuMs: 5000, liveChildCount: 0})})
|
|
276
|
-
|
|
277
|
-
expect(classify({current}).status).toBe("wedged")
|
|
278
|
-
expect(escalate({current}).decision).toBe("report")
|
|
279
|
-
expect(escalate({current}).reason).toBe("no escalation: liveness window 0ms is below the 20000ms minimum")
|
|
280
|
-
expect(shortWindow.reason).toBe("no escalation: liveness window 1000ms is below the 20000ms minimum")
|
|
281
|
-
})
|
|
282
|
-
|
|
283
|
-
test("admission allows a handoff that stays within the bound and counts the incumbent", () => {
|
|
284
|
-
const admission = admitGenerationRelease({
|
|
285
|
-
candidateReleaseId: "v3",
|
|
286
|
-
draining: [{reason: "1 live connection still owned by the generation", releaseId: "v1", status: "progressing"}],
|
|
287
|
-
incumbentWillDrain: true,
|
|
288
|
-
maxConcurrentDrainingReleaseGroups: 2
|
|
289
|
-
})
|
|
290
|
-
|
|
291
|
-
expect(admission.allowed).toBe(true)
|
|
292
|
-
expect(admission.wouldBeDrainingCount).toBe(2)
|
|
293
|
-
expect(admission.reason).toBe(undefined)
|
|
294
|
-
})
|
|
295
|
-
|
|
296
|
-
test("admission refuses before activation and names every draining group with its classification", () => {
|
|
297
|
-
const admission = admitGenerationRelease({
|
|
298
|
-
candidateReleaseId: "20261115073717",
|
|
299
|
-
draining: [
|
|
300
|
-
{reason: "1 live process with 3 runner children, no live connection, every retirement budget expired, CPU flat across 25000ms", releaseId: "20261115063805", status: "wedged"},
|
|
301
|
-
{reason: "2 live connections still owned by the generation", releaseId: "20261114232900", status: "progressing"}
|
|
302
|
-
],
|
|
303
|
-
incumbentWillDrain: true,
|
|
304
|
-
maxConcurrentDrainingReleaseGroups: 2
|
|
305
|
-
})
|
|
306
|
-
|
|
307
|
-
expect(admission.allowed).toBe(false)
|
|
308
|
-
expect(admission.wouldBeDrainingCount).toBe(3)
|
|
309
|
-
expect(admission.reason).toContain("Deploy refused before activation: 2 release groups already draining")
|
|
310
|
-
expect(admission.reason).toContain("20261115063805 wedged")
|
|
311
|
-
expect(admission.reason).toContain("Activating 20261115073717 would make 3, above the releaseRetention.maxConcurrentDrainingReleaseGroups bound of 2")
|
|
312
|
-
})
|
|
313
|
-
|
|
314
|
-
test("a first deploy with no incumbent adds no draining group", () => {
|
|
315
|
-
const admission = admitGenerationRelease({candidateReleaseId: "v1", draining: [], incumbentWillDrain: false, maxConcurrentDrainingReleaseGroups: 1})
|
|
316
|
-
|
|
317
|
-
expect(admission.allowed).toBe(true)
|
|
318
|
-
expect(admission.wouldBeDrainingCount).toBe(0)
|
|
319
|
-
})
|
|
320
|
-
|
|
321
|
-
test("a bound of 0 disables the admission check", () => {
|
|
322
|
-
const draining = Array.from({length: 30}, (_, index) => ({reason: "no runner child remaining", releaseId: `v${index}`, status: /** @type {const} */ ("wedged")}))
|
|
323
|
-
const admission = admitGenerationRelease({candidateReleaseId: "v31", draining, incumbentWillDrain: true, maxConcurrentDrainingReleaseGroups: 0})
|
|
324
|
-
|
|
325
|
-
expect(admission.allowed).toBe(true)
|
|
326
|
-
expect(admission.wouldBeDrainingCount).toBe(31)
|
|
327
|
-
})
|
|
328
|
-
})
|
|
@@ -1,106 +0,0 @@
|
|
|
1
|
-
// @ts-check
|
|
2
|
-
|
|
3
|
-
import http from "node:http"
|
|
4
|
-
import net from "node:net"
|
|
5
|
-
|
|
6
|
-
const port = Number(requiredEnv("ROLLBRIDGE_PORT"))
|
|
7
|
-
const bootMs = Number(process.env.ROLLBRIDGE_BOOT_MS || "0")
|
|
8
|
-
const dependencyPort = process.env.ROLLBRIDGE_DEPENDENCY_PORT
|
|
9
|
-
const releaseId = process.env.ROLLBRIDGE_RELEASE_ID || "unknown"
|
|
10
|
-
|
|
11
|
-
log(`starting (boot budget ${bootMs}ms)`)
|
|
12
|
-
|
|
13
|
-
if (dependencyPort) {
|
|
14
|
-
// The app performs its own dependency wait, exactly as the production web command does.
|
|
15
|
-
await waitForDependency(Number(dependencyPort))
|
|
16
|
-
log(`dependency ${dependencyPort} accepted a connection`)
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
await sleep(bootMs)
|
|
20
|
-
|
|
21
|
-
const server = http.createServer((request, response) => {
|
|
22
|
-
if (request.url === "/ping") {
|
|
23
|
-
response.writeHead(200, {"Content-Type": "application/json"})
|
|
24
|
-
response.end(JSON.stringify({message: "Pong", releaseId}))
|
|
25
|
-
return
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
response.writeHead(200, {"Content-Type": "text/plain; charset=utf-8"})
|
|
29
|
-
response.end(`${releaseId}\n`)
|
|
30
|
-
})
|
|
31
|
-
|
|
32
|
-
process.on("SIGTERM", () => {
|
|
33
|
-
server.close(() => process.exit(0))
|
|
34
|
-
setTimeout(() => process.exit(0), 10).unref()
|
|
35
|
-
})
|
|
36
|
-
|
|
37
|
-
server.listen(port, "127.0.0.1")
|
|
38
|
-
log(`listening on ${port}`)
|
|
39
|
-
|
|
40
|
-
/**
|
|
41
|
-
* @param {number} dependencyPortNumber - Port of the process this app depends on.
|
|
42
|
-
* @returns {Promise<void>} Resolves once the dependency accepts a connection.
|
|
43
|
-
*/
|
|
44
|
-
async function waitForDependency(dependencyPortNumber) {
|
|
45
|
-
const deadline = Date.now() + 10000
|
|
46
|
-
|
|
47
|
-
while (Date.now() < deadline) {
|
|
48
|
-
if (await tryConnect(dependencyPortNumber)) return
|
|
49
|
-
await sleep(50)
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
log(`dependency ${dependencyPortNumber} never accepted a connection`)
|
|
53
|
-
process.exit(21)
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
/**
|
|
57
|
-
* @param {number} dependencyPortNumber - Port to connect to.
|
|
58
|
-
* @returns {Promise<boolean>} True when the dependency accepted the connection.
|
|
59
|
-
*/
|
|
60
|
-
function tryConnect(dependencyPortNumber) {
|
|
61
|
-
return new Promise((resolve) => {
|
|
62
|
-
const socket = net.createConnection({host: "127.0.0.1", port: dependencyPortNumber})
|
|
63
|
-
let settled = false
|
|
64
|
-
|
|
65
|
-
/** @param {boolean} connected - Whether the connection succeeded. */
|
|
66
|
-
const finish = (connected) => {
|
|
67
|
-
if (settled) return
|
|
68
|
-
settled = true
|
|
69
|
-
socket.destroy()
|
|
70
|
-
resolve(connected)
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
socket.setTimeout(200)
|
|
74
|
-
socket.once("connect", () => finish(true))
|
|
75
|
-
socket.once("error", () => finish(false))
|
|
76
|
-
socket.once("timeout", () => finish(false))
|
|
77
|
-
})
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
/**
|
|
81
|
-
* @param {number} ms - Milliseconds.
|
|
82
|
-
* @returns {Promise<void>} Resolves after the delay.
|
|
83
|
-
*/
|
|
84
|
-
function sleep(ms) {
|
|
85
|
-
return new Promise((resolve) => setTimeout(resolve, ms))
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
/**
|
|
89
|
-
* @param {string} line - Output line, so the failure record can name the app's last output.
|
|
90
|
-
* @returns {void}
|
|
91
|
-
*/
|
|
92
|
-
function log(line) {
|
|
93
|
-
console.log(`boot: ${process.pid} ${line}`)
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
/**
|
|
97
|
-
* @param {string} key - Environment variable name.
|
|
98
|
-
* @returns {string} Environment variable value.
|
|
99
|
-
*/
|
|
100
|
-
function requiredEnv(key) {
|
|
101
|
-
const value = process.env[key]
|
|
102
|
-
|
|
103
|
-
if (!value) throw new Error(`${key} is required`)
|
|
104
|
-
|
|
105
|
-
return value
|
|
106
|
-
}
|
|
@@ -1,50 +0,0 @@
|
|
|
1
|
-
// @ts-check
|
|
2
|
-
|
|
3
|
-
import http from "node:http"
|
|
4
|
-
import fs from "node:fs"
|
|
5
|
-
|
|
6
|
-
const port = Number(requiredEnv("ROLLBRIDGE_PORT"))
|
|
7
|
-
const gateArg = process.argv.find((arg) => arg.startsWith("--gate="))
|
|
8
|
-
|
|
9
|
-
if (!gateArg) throw new Error("--gate=<path> is required")
|
|
10
|
-
|
|
11
|
-
const gatePath = gateArg.slice("--gate=".length)
|
|
12
|
-
const releaseId = process.env.ROLLBRIDGE_RELEASE_ID || "unknown"
|
|
13
|
-
// v3 stays unhealthy until the gate file appears, so its deploy fails at health.
|
|
14
|
-
const unhealthy = releaseId === "v3" && !fs.existsSync(gatePath)
|
|
15
|
-
|
|
16
|
-
const server = http.createServer((request, response) => {
|
|
17
|
-
if (unhealthy) {
|
|
18
|
-
response.writeHead(500)
|
|
19
|
-
response.end("bad")
|
|
20
|
-
return
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
if (request.url === "/ping") {
|
|
24
|
-
response.writeHead(200, {"Content-Type": "application/json"})
|
|
25
|
-
response.end(JSON.stringify({message: "Pong", releaseId}))
|
|
26
|
-
return
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
response.writeHead(200, {"Content-Type": "text/plain; charset=utf-8"})
|
|
30
|
-
response.end(`${releaseId}\n`)
|
|
31
|
-
})
|
|
32
|
-
|
|
33
|
-
process.on("SIGTERM", () => {
|
|
34
|
-
server.close(() => process.exit(0))
|
|
35
|
-
setTimeout(() => process.exit(0), 10).unref()
|
|
36
|
-
})
|
|
37
|
-
|
|
38
|
-
server.listen(port, "127.0.0.1")
|
|
39
|
-
|
|
40
|
-
/**
|
|
41
|
-
* @param {string} key - Environment variable name.
|
|
42
|
-
* @returns {string} Environment variable value.
|
|
43
|
-
*/
|
|
44
|
-
function requiredEnv(key) {
|
|
45
|
-
const value = process.env[key]
|
|
46
|
-
|
|
47
|
-
if (!value) throw new Error(`${key} is required`)
|
|
48
|
-
|
|
49
|
-
return value
|
|
50
|
-
}
|