rollbridge 0.1.62 → 0.1.65
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.d/20261113-bounded-nonblocking-retirement.md +5 -0
- package/changelog.d/20261115-bound-draining-release-groups.md +62 -0
- package/docs/cli.md +44 -0
- package/docs/compose-services.md +8 -6
- package/docs/config.md +27 -1
- package/package.json +1 -1
- package/src/config.js +15 -3
- package/src/daemon.js +83 -1
- package/src/drain-policy.js +159 -0
- package/src/managed-process.js +116 -9
- package/src/process-guardian.js +4 -5
- package/src/process-memory.js +62 -2
- package/src/release-group.js +110 -5
- package/test/config-validation.test.js +11 -4
- package/test/drain-policy.test.js +199 -0
- package/test/managed-process.test.js +47 -1
- package/test/process-memory.test.js +47 -1
- package/test/release-group.test.js +155 -2
- package/test/rollbridge.test.js +120 -0
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
import {describe, expect, test} from "@velocious/testing"
|
|
4
|
+
import {admitGenerationRelease, classifyDrainProgress} from "../src/drain-policy.js"
|
|
5
|
+
|
|
6
|
+
describe("drain-policy", () => {
|
|
7
|
+
|
|
8
|
+
const WINDOW_MS = 20000
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* @param {object} [options] - Observation shape.
|
|
12
|
+
* @param {number} [options.atMs] - Observation time.
|
|
13
|
+
* @param {number | undefined} [options.cpuMs] - Cumulative CPU of the group.
|
|
14
|
+
* @param {number | undefined} [options.heldProcessGroupPgid] - Group still alive after its tracked process exited.
|
|
15
|
+
* @param {number} [options.liveChildCount] - Runner children other than the group leader.
|
|
16
|
+
* @param {number | undefined} [options.pid] - Live pid, or undefined when the process is gone.
|
|
17
|
+
* @param {boolean} [options.retirementTimedOut] - Whether the supervisor already gave up on it.
|
|
18
|
+
* @returns {import("../src/drain-policy.js").DrainLivenessSample} One liveness observation.
|
|
19
|
+
*/
|
|
20
|
+
function sample(options = {}) {
|
|
21
|
+
return {
|
|
22
|
+
atMs: options.atMs ?? 1000000,
|
|
23
|
+
processes: [{
|
|
24
|
+
cpuMs: "cpuMs" in options ? options.cpuMs : 5000,
|
|
25
|
+
heldProcessGroupPgid: options.heldProcessGroupPgid,
|
|
26
|
+
id: "worker",
|
|
27
|
+
liveChildCount: options.liveChildCount ?? 1,
|
|
28
|
+
pid: "pid" in options ? options.pid : 4242,
|
|
29
|
+
retirementTimedOut: options.retirementTimedOut ?? true
|
|
30
|
+
}]
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* @param {object} [options] - Classification inputs.
|
|
36
|
+
* @param {number} [options.connectionCount] - Live connections owned by the generation.
|
|
37
|
+
* @param {import("../src/drain-policy.js").DrainLivenessSample} [options.current] - Newest observation.
|
|
38
|
+
* @param {import("../src/drain-policy.js").DrainLivenessSample | undefined} [options.previous] - Older observation.
|
|
39
|
+
* @returns {import("../src/drain-policy.js").DrainClassification} Classification.
|
|
40
|
+
*/
|
|
41
|
+
function classify(options = {}) {
|
|
42
|
+
return classifyDrainProgress({
|
|
43
|
+
connectionCount: options.connectionCount ?? 0,
|
|
44
|
+
currentSample: options.current ?? sample(),
|
|
45
|
+
minSampleWindowMs: WINDOW_MS,
|
|
46
|
+
previousSample: options.previous
|
|
47
|
+
})
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
test("a generation that still owns connections is draining, never wedged", () => {
|
|
51
|
+
const result = classify({connectionCount: 3, current: sample({liveChildCount: 0})})
|
|
52
|
+
|
|
53
|
+
expect(result.status).toBe("progressing")
|
|
54
|
+
expect(result.reason).toBe("3 live connections still owned by the generation")
|
|
55
|
+
})
|
|
56
|
+
|
|
57
|
+
test("a process still inside its retirement budget is draining", () => {
|
|
58
|
+
const result = classify({current: sample({liveChildCount: 0, retirementTimedOut: false})})
|
|
59
|
+
|
|
60
|
+
expect(result.status).toBe("progressing")
|
|
61
|
+
expect(result.reason).toBe("process worker is still inside its retirement budget")
|
|
62
|
+
})
|
|
63
|
+
|
|
64
|
+
test("a live process past its retirement budget with no runner child left is wedged", () => {
|
|
65
|
+
const result = classify({current: sample({liveChildCount: 0})})
|
|
66
|
+
|
|
67
|
+
expect(result.status).toBe("wedged")
|
|
68
|
+
expect(result.reason).toBe("1 live process past its retirement budget with no runner child remaining")
|
|
69
|
+
})
|
|
70
|
+
|
|
71
|
+
test("flat CPU across a sufficient window is wedged, and the window is reported", () => {
|
|
72
|
+
const previous = sample({atMs: 1000000, cpuMs: 5000})
|
|
73
|
+
const current = sample({atMs: 1025000, cpuMs: 5000, liveChildCount: 3})
|
|
74
|
+
const result = classify({current, previous})
|
|
75
|
+
|
|
76
|
+
expect(result.status).toBe("wedged")
|
|
77
|
+
expect(result.sampleWindowMs).toBe(25000)
|
|
78
|
+
expect(result.cpuDeltaMs).toBe(0)
|
|
79
|
+
expect(result.reason).toBe("1 live process with 3 runner children, no live connection, every retirement budget expired, CPU flat across 25000ms")
|
|
80
|
+
})
|
|
81
|
+
|
|
82
|
+
test("CPU that advanced across the window is a progressing drain", () => {
|
|
83
|
+
const previous = sample({atMs: 1000000, cpuMs: 5000})
|
|
84
|
+
const current = sample({atMs: 1025000, cpuMs: 9000})
|
|
85
|
+
const result = classify({current, previous})
|
|
86
|
+
|
|
87
|
+
expect(result.status).toBe("progressing")
|
|
88
|
+
expect(result.cpuDeltaMs).toBe(4000)
|
|
89
|
+
expect(result.reason).toBe("CPU advanced 4000ms across 25000ms")
|
|
90
|
+
})
|
|
91
|
+
|
|
92
|
+
test("CPU leaving the group with an exited member is a progressing drain", () => {
|
|
93
|
+
const previous = sample({atMs: 1000000, cpuMs: 9000})
|
|
94
|
+
const current = sample({atMs: 1025000, cpuMs: 5000})
|
|
95
|
+
const result = classify({current, previous})
|
|
96
|
+
|
|
97
|
+
expect(result.status).toBe("progressing")
|
|
98
|
+
expect(result.cpuDeltaMs).toBe(-4000)
|
|
99
|
+
expect(result.reason).toBe("the group changed (1 live process, -4000ms of consumed CPU left the group) across 25000ms")
|
|
100
|
+
})
|
|
101
|
+
|
|
102
|
+
test("the first observation cannot claim flatness", () => {
|
|
103
|
+
const result = classify({current: sample()})
|
|
104
|
+
|
|
105
|
+
expect(result.status).toBe("unproven")
|
|
106
|
+
expect(result.sampleWindowMs).toBe(0)
|
|
107
|
+
expect(result.reason).toBe("first liveness observation; a 20000ms window is required before CPU flatness can be claimed")
|
|
108
|
+
})
|
|
109
|
+
|
|
110
|
+
test("a window shorter than the minimum cannot claim flatness", () => {
|
|
111
|
+
const previous = sample({atMs: 1000000})
|
|
112
|
+
const current = sample({atMs: 1010000})
|
|
113
|
+
const result = classify({current, previous})
|
|
114
|
+
|
|
115
|
+
expect(result.status).toBe("unproven")
|
|
116
|
+
expect(result.sampleWindowMs).toBe(10000)
|
|
117
|
+
expect(result.reason).toBe("liveness window 10000ms is below the 20000ms minimum")
|
|
118
|
+
})
|
|
119
|
+
|
|
120
|
+
test("an unmeasurable CPU counter is unproven, never wedged", () => {
|
|
121
|
+
const previous = sample({atMs: 1000000, cpuMs: undefined})
|
|
122
|
+
const current = sample({atMs: 1025000, cpuMs: undefined})
|
|
123
|
+
const result = classify({current, previous})
|
|
124
|
+
|
|
125
|
+
expect(result.status).toBe("unproven")
|
|
126
|
+
expect(result.cpuDeltaMs).toBe(undefined)
|
|
127
|
+
expect(result.reason).toBe("process CPU is not measurable on this host")
|
|
128
|
+
})
|
|
129
|
+
|
|
130
|
+
test("a generation with no live process left is not a wedge", () => {
|
|
131
|
+
const result = classify({current: sample({pid: undefined})})
|
|
132
|
+
|
|
133
|
+
expect(result.status).toBe("progressing")
|
|
134
|
+
expect(result.reason).toBe("no live process remains in the generation")
|
|
135
|
+
})
|
|
136
|
+
|
|
137
|
+
test("a process group that outlived its tracked process is still a live process of the generation", () => {
|
|
138
|
+
const result = classify({current: sample({heldProcessGroupPgid: 4242, liveChildCount: 0, pid: undefined})})
|
|
139
|
+
|
|
140
|
+
expect(result.status).toBe("wedged")
|
|
141
|
+
expect(result.reason).toBe("1 live process past its retirement budget with no runner child remaining; 1 tracked process already exited and left worker pgid 4242 alive")
|
|
142
|
+
})
|
|
143
|
+
|
|
144
|
+
test("a held process group that still consumes CPU across the window is a progressing drain", () => {
|
|
145
|
+
const previous = sample({atMs: 1000000, cpuMs: 5000, heldProcessGroupPgid: 4242, liveChildCount: 2, pid: undefined})
|
|
146
|
+
const current = sample({atMs: 1025000, cpuMs: 9000, heldProcessGroupPgid: 4242, liveChildCount: 2, pid: undefined})
|
|
147
|
+
const result = classify({current, previous})
|
|
148
|
+
|
|
149
|
+
expect(result.status).toBe("progressing")
|
|
150
|
+
expect(result.cpuDeltaMs).toBe(4000)
|
|
151
|
+
expect(result.reason).toBe("CPU advanced 4000ms across 25000ms")
|
|
152
|
+
})
|
|
153
|
+
|
|
154
|
+
test("admission allows a handoff that stays within the bound and counts the incumbent", () => {
|
|
155
|
+
const admission = admitGenerationRelease({
|
|
156
|
+
candidateReleaseId: "v3",
|
|
157
|
+
draining: [{reason: "1 live connection still owned by the generation", releaseId: "v1", status: "progressing"}],
|
|
158
|
+
incumbentWillDrain: true,
|
|
159
|
+
maxConcurrentDrainingReleaseGroups: 2
|
|
160
|
+
})
|
|
161
|
+
|
|
162
|
+
expect(admission.allowed).toBe(true)
|
|
163
|
+
expect(admission.wouldBeDrainingCount).toBe(2)
|
|
164
|
+
expect(admission.reason).toBe(undefined)
|
|
165
|
+
})
|
|
166
|
+
|
|
167
|
+
test("admission refuses before activation and names every draining group with its classification", () => {
|
|
168
|
+
const admission = admitGenerationRelease({
|
|
169
|
+
candidateReleaseId: "20261115073717",
|
|
170
|
+
draining: [
|
|
171
|
+
{reason: "1 live process with 3 runner children, no live connection, every retirement budget expired, CPU flat across 25000ms", releaseId: "20261115063805", status: "wedged"},
|
|
172
|
+
{reason: "2 live connections still owned by the generation", releaseId: "20261114232900", status: "progressing"}
|
|
173
|
+
],
|
|
174
|
+
incumbentWillDrain: true,
|
|
175
|
+
maxConcurrentDrainingReleaseGroups: 2
|
|
176
|
+
})
|
|
177
|
+
|
|
178
|
+
expect(admission.allowed).toBe(false)
|
|
179
|
+
expect(admission.wouldBeDrainingCount).toBe(3)
|
|
180
|
+
expect(admission.reason).toContain("Deploy refused before activation: 2 release groups already draining")
|
|
181
|
+
expect(admission.reason).toContain("20261115063805 wedged")
|
|
182
|
+
expect(admission.reason).toContain("Activating 20261115073717 would make 3, above the releaseRetention.maxConcurrentDrainingReleaseGroups bound of 2")
|
|
183
|
+
})
|
|
184
|
+
|
|
185
|
+
test("a first deploy with no incumbent adds no draining group", () => {
|
|
186
|
+
const admission = admitGenerationRelease({candidateReleaseId: "v1", draining: [], incumbentWillDrain: false, maxConcurrentDrainingReleaseGroups: 1})
|
|
187
|
+
|
|
188
|
+
expect(admission.allowed).toBe(true)
|
|
189
|
+
expect(admission.wouldBeDrainingCount).toBe(0)
|
|
190
|
+
})
|
|
191
|
+
|
|
192
|
+
test("a bound of 0 disables the admission check", () => {
|
|
193
|
+
const draining = Array.from({length: 30}, (_, index) => ({reason: "no runner child remaining", releaseId: `v${index}`, status: /** @type {const} */ ("wedged")}))
|
|
194
|
+
const admission = admitGenerationRelease({candidateReleaseId: "v31", draining, incumbentWillDrain: true, maxConcurrentDrainingReleaseGroups: 0})
|
|
195
|
+
|
|
196
|
+
expect(admission.allowed).toBe(true)
|
|
197
|
+
expect(admission.wouldBeDrainingCount).toBe(31)
|
|
198
|
+
})
|
|
199
|
+
})
|
|
@@ -5,7 +5,7 @@ import os from "node:os"
|
|
|
5
5
|
import path from "node:path"
|
|
6
6
|
import {describe, expect, test} from "@velocious/testing"
|
|
7
7
|
import {fileURLToPath} from "node:url"
|
|
8
|
-
import ManagedProcess from "../src/managed-process.js"
|
|
8
|
+
import ManagedProcess, {childRecordIsGone} from "../src/managed-process.js"
|
|
9
9
|
|
|
10
10
|
describe("managed-process", () => {
|
|
11
11
|
|
|
@@ -308,6 +308,42 @@ test("runs quiet and drain lifecycle hooks before stopping", async () => {
|
|
|
308
308
|
}
|
|
309
309
|
})
|
|
310
310
|
|
|
311
|
+
test("retirement timeout leaves an indefinite worker alive without SIGKILL", async () => {
|
|
312
|
+
/** @type {string[]} */
|
|
313
|
+
const signals = []
|
|
314
|
+
const managed = new ManagedProcess({
|
|
315
|
+
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify("process.on('SIGTERM', () => {}); setInterval(() => {}, 1000)")}`,
|
|
316
|
+
cwd: undefined,
|
|
317
|
+
env: {},
|
|
318
|
+
id: "worker",
|
|
319
|
+
lifecycle: {drainTimeoutMs: 0},
|
|
320
|
+
logger: () => {},
|
|
321
|
+
outputLines: 50,
|
|
322
|
+
restartDelayMs: 10,
|
|
323
|
+
shouldRestart: () => false,
|
|
324
|
+
stopSignal: "SIGTERM",
|
|
325
|
+
stopTimeoutMs: "indefinite"
|
|
326
|
+
})
|
|
327
|
+
const signalProcessGroup = managed.signalProcessGroup.bind(managed)
|
|
328
|
+
managed.signalProcessGroup = async (signal, pgid, deadline) => {
|
|
329
|
+
signals.push(signal)
|
|
330
|
+
await signalProcessGroup(signal, pgid, deadline)
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
try {
|
|
334
|
+
await managed.start()
|
|
335
|
+
await managed.stop({retirementTimeoutMs: 25})
|
|
336
|
+
|
|
337
|
+
expect(managed.retirementTimedOut).toBe(true)
|
|
338
|
+
expect(signals).toEqual(["SIGTERM"])
|
|
339
|
+
expect(managed.status().pid).toBeDefined()
|
|
340
|
+
} finally {
|
|
341
|
+
const pid = managed.status().pid
|
|
342
|
+
if (pid) process.kill(-pid, "SIGKILL")
|
|
343
|
+
if (managed.exitPromise) await managed.exitPromise
|
|
344
|
+
}
|
|
345
|
+
})
|
|
346
|
+
|
|
311
347
|
test("a configured stopCommand is used instead of the stop signal", async () => {
|
|
312
348
|
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-hooks-"))
|
|
313
349
|
const logPath = path.join(dir, "stop.log")
|
|
@@ -894,4 +930,14 @@ test("the unlimited constant-delay fast path still applies maxDelayMs", () => {
|
|
|
894
930
|
|
|
895
931
|
expect(queued).toBe(5)
|
|
896
932
|
})
|
|
933
|
+
|
|
934
|
+
test("the guardian fence releases only a record that is terminal and holds no process group", () => {
|
|
935
|
+
expect(childRecordIsGone({heldProcessGroup: undefined, pid: undefined, state: "stopped"})).toBe(true)
|
|
936
|
+
expect(childRecordIsGone({heldProcessGroup: undefined, pid: undefined, state: "failed"})).toBe(true)
|
|
937
|
+
expect(childRecordIsGone({heldProcessGroup: undefined, pid: 4242, state: "running"})).toBe(false)
|
|
938
|
+
|
|
939
|
+
// The shape a drained generation leaves behind: the tracked process record is terminal, but the
|
|
940
|
+
// group it led still holds live members, so the key is still held by that generation.
|
|
941
|
+
expect(childRecordIsGone({heldProcessGroup: {liveMemberCount: 1, pgid: 4242}, pid: undefined, state: "stopped"})).toBe(false)
|
|
942
|
+
})
|
|
897
943
|
})
|
|
@@ -4,7 +4,7 @@ import fs from "node:fs"
|
|
|
4
4
|
import {describe, expect, test} from "@velocious/testing"
|
|
5
5
|
import os from "node:os"
|
|
6
6
|
import path from "node:path"
|
|
7
|
-
import {measureProcessGroupRssBytes, processGroupHasLiveMembers, processGroupMembers} from "../src/process-memory.js"
|
|
7
|
+
import {measureProcessGroupRssBytes, processGroupHasLiveMembers, processGroupMembers, sampleProcessGroup} from "../src/process-memory.js"
|
|
8
8
|
|
|
9
9
|
describe("process-memory", () => {
|
|
10
10
|
|
|
@@ -59,4 +59,50 @@ test("treats a process group containing only defunct members as stopped", () =>
|
|
|
59
59
|
fs.rmSync(procPath, {force: true, recursive: true})
|
|
60
60
|
}
|
|
61
61
|
})
|
|
62
|
+
|
|
63
|
+
test("samples a process group's cumulative CPU and its live member count", () => {
|
|
64
|
+
const procPath = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-proc-cpu-"))
|
|
65
|
+
|
|
66
|
+
try {
|
|
67
|
+
// utime and stime are stat fields 14 and 15: index 11 and 12 after the parenthesised comm,
|
|
68
|
+
// so exactly eight placeholder fields sit between pgrp and utime.
|
|
69
|
+
fs.mkdirSync(path.join(procPath, "101"))
|
|
70
|
+
fs.writeFileSync(path.join(procPath, "101", "stat"), "101 (worker) S 1 77 0 0 0 0 0 0 0 0 300 200")
|
|
71
|
+
fs.mkdirSync(path.join(procPath, "102"))
|
|
72
|
+
fs.writeFileSync(path.join(procPath, "102", "stat"), "102 (runner) Z 1 77 0 0 0 0 0 0 0 0 10 5")
|
|
73
|
+
fs.mkdirSync(path.join(procPath, "103"))
|
|
74
|
+
fs.writeFileSync(path.join(procPath, "103", "stat"), "103 (other) S 1 99 0 0 0 0 0 0 0 0 9000 9000")
|
|
75
|
+
|
|
76
|
+
const sampled = sampleProcessGroup(77, procPath)
|
|
77
|
+
|
|
78
|
+
// Defunct members keep their consumed CPU, so the total covers both group members and
|
|
79
|
+
// excludes the foreign group 99 entirely: (300 + 200 + 10 + 5) ticks * 10ms per tick.
|
|
80
|
+
expect(sampled).toEqual({cpuMs: 5150, liveMemberCount: 1, memberCount: 2})
|
|
81
|
+
} finally {
|
|
82
|
+
fs.rmSync(procPath, {force: true, recursive: true})
|
|
83
|
+
}
|
|
84
|
+
})
|
|
85
|
+
|
|
86
|
+
test("reports group counts when a member's CPU fields are unreadable", () => {
|
|
87
|
+
const procPath = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-proc-cpu-partial-"))
|
|
88
|
+
|
|
89
|
+
try {
|
|
90
|
+
fs.mkdirSync(path.join(procPath, "101"))
|
|
91
|
+
fs.writeFileSync(path.join(procPath, "101", "stat"), "101 (worker) S 1 77 0 0")
|
|
92
|
+
|
|
93
|
+
expect(sampleProcessGroup(77, procPath)).toEqual({cpuMs: undefined, liveMemberCount: 1, memberCount: 1})
|
|
94
|
+
} finally {
|
|
95
|
+
fs.rmSync(procPath, {force: true, recursive: true})
|
|
96
|
+
}
|
|
97
|
+
})
|
|
98
|
+
|
|
99
|
+
test("reports a zero sample for a process group with no members", () => {
|
|
100
|
+
expect(sampleProcessGroup(2147483646)).toEqual({cpuMs: 0, liveMemberCount: 0, memberCount: 0})
|
|
101
|
+
})
|
|
102
|
+
|
|
103
|
+
linuxTest("samples the current process group's CPU as a positive measurable total", () => {
|
|
104
|
+
const sampled = sampleProcessGroup(currentProcessGroupId())
|
|
105
|
+
|
|
106
|
+
expect({value: Boolean(sampled && typeof sampled.cpuMs === "number" && sampled.cpuMs >= 0 && sampled.liveMemberCount >= 1), context: `expected a sampled group, got ${JSON.stringify(sampled)}`}).toMatchObject({value: true})
|
|
107
|
+
})
|
|
62
108
|
})
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
3
|
import {describe, expect, test} from "@velocious/testing"
|
|
4
|
+
import fs from "node:fs"
|
|
5
|
+
import os from "node:os"
|
|
6
|
+
import path from "node:path"
|
|
4
7
|
import ReleaseGroup from "../src/release-group.js"
|
|
5
8
|
import {normalizeConfig} from "../src/config.js"
|
|
6
9
|
|
|
@@ -9,9 +12,10 @@ describe("release-group", () => {
|
|
|
9
12
|
/**
|
|
10
13
|
* @param {import("../src/json.js").JsonValue} webProcess - The single proxied process definition.
|
|
11
14
|
* @param {() => boolean} [shouldStart] - Whether process starts remain allowed.
|
|
15
|
+
* @param {string} [procPath] - Procfs root for drain liveness observations.
|
|
12
16
|
* @returns {ReleaseGroup} A release group ready for buildProcess.
|
|
13
17
|
*/
|
|
14
|
-
function buildRelease(webProcess, shouldStart = () => true) {
|
|
18
|
+
function buildRelease(webProcess, shouldStart = () => true, procPath = "/proc") {
|
|
15
19
|
const config = normalizeConfig({
|
|
16
20
|
application: "demo",
|
|
17
21
|
control: {path: "/tmp/rollbridge-release-group.sock"},
|
|
@@ -19,7 +23,37 @@ function buildRelease(webProcess, shouldStart = () => true) {
|
|
|
19
23
|
proxy: {host: "127.0.0.1", port: 0}
|
|
20
24
|
})
|
|
21
25
|
|
|
22
|
-
return new ReleaseGroup({config, logger: () => {}, releaseId: "v1", releasePath: "/tmp/rel", revision: "v1", shouldStart})
|
|
26
|
+
return new ReleaseGroup({config, logger: () => {}, procPath, releaseId: "v1", releasePath: "/tmp/rel", revision: "v1", shouldStart})
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* @param {string} procPath - Procfs root.
|
|
31
|
+
* @param {number} pid - Process id.
|
|
32
|
+
* @param {number} pgrp - Process group id.
|
|
33
|
+
* @param {string} state - Process state letter.
|
|
34
|
+
* @param {number} utimeTicks - Cumulative user CPU in clock ticks.
|
|
35
|
+
* @param {number} stimeTicks - Cumulative system CPU in clock ticks.
|
|
36
|
+
* @returns {void} One synthetic procfs member.
|
|
37
|
+
*/
|
|
38
|
+
function writeProcMember(procPath, pid, pgrp, state, utimeTicks, stimeTicks) {
|
|
39
|
+
fs.mkdirSync(path.join(procPath, String(pid)), {recursive: true})
|
|
40
|
+
|
|
41
|
+
// Eight placeholder fields sit between pgrp and utime, which is stat field 14 (index 11
|
|
42
|
+
// after the parenthesised comm); stime is field 15 (index 12).
|
|
43
|
+
fs.writeFileSync(path.join(procPath, String(pid), "stat"), `${pid} (proc) ${state} 1 ${pgrp} 0 0 0 0 0 0 0 0 ${utimeTicks} ${stimeTicks}`)
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* @param {number | undefined} pid - Live pid, or undefined when the tracked process is gone.
|
|
48
|
+
* @param {import("../src/managed-process.js").HeldProcessGroup | undefined} [heldProcessGroup] - Group still alive after the tracked process exited.
|
|
49
|
+
* @returns {import("../src/managed-process.js").default} A process whose record is already terminal.
|
|
50
|
+
*/
|
|
51
|
+
function drainedProcess(pid, heldProcessGroup = undefined) {
|
|
52
|
+
// eslint-disable-next-line local/no-unknown-jsdoc-type
|
|
53
|
+
return /** @type {import("../src/managed-process.js").default} */ (/** @type {unknown} */ ({
|
|
54
|
+
retirementTimedOut: true,
|
|
55
|
+
status: () => ({heldProcessGroup, pid, retirementTimedOut: true, state: "stopped"})
|
|
56
|
+
}))
|
|
23
57
|
}
|
|
24
58
|
|
|
25
59
|
test("templates interpolate values from the daemon environment", () => {
|
|
@@ -109,6 +143,34 @@ test("stopping a release abandons its connection count instead of persisting a s
|
|
|
109
143
|
expect(release.localConnections()).toEqual({http: 0, websocket: 0})
|
|
110
144
|
})
|
|
111
145
|
|
|
146
|
+
test("a wedged non-blocking process cannot block release retirement", async () => {
|
|
147
|
+
/** @type {Array<{message: string, data?: import("../src/json.js").JsonValue}>} */
|
|
148
|
+
const logs = []
|
|
149
|
+
const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}})
|
|
150
|
+
release.logger = (message, data) => logs.push({message, data})
|
|
151
|
+
release.state = "draining"
|
|
152
|
+
release.nonBlockingDrainIds.add("worker")
|
|
153
|
+
release.handoffServiceIds.add("jobs-main")
|
|
154
|
+
// eslint-disable-next-line local/no-unknown-jsdoc-type
|
|
155
|
+
release.processes.set("worker", /** @type {import("../src/managed-process.js").default} */ (/** @type {unknown} */ ({
|
|
156
|
+
quiesceStrict: async () => {},
|
|
157
|
+
status: () => ({pid: 4242}),
|
|
158
|
+
retirementTimedOut: true,
|
|
159
|
+
stop: async () => {}
|
|
160
|
+
})))
|
|
161
|
+
let handoffStopped = false
|
|
162
|
+
// eslint-disable-next-line local/no-unknown-jsdoc-type
|
|
163
|
+
release.processes.set("jobs-main", /** @type {import("../src/managed-process.js").default} */ (/** @type {unknown} */ ({
|
|
164
|
+
stop: async () => { handoffStopped = true }
|
|
165
|
+
})))
|
|
166
|
+
|
|
167
|
+
await release.drainAndStop(0, {...release.config, proxy: {...release.config.proxy, nonBlockingDrainTimeoutMs: 20}})
|
|
168
|
+
|
|
169
|
+
expect(release.state).toBe("stopped")
|
|
170
|
+
expect(handoffStopped).toBe(true)
|
|
171
|
+
expect(logs).toMatchObject([{message: "release non-blocking process still draining", data: {pid: 4242, processId: "worker"}}])
|
|
172
|
+
})
|
|
173
|
+
|
|
112
174
|
test("restoring a stopped release does not re-adopt a stale connection count", async () => {
|
|
113
175
|
const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}})
|
|
114
176
|
|
|
@@ -118,6 +180,7 @@ test("restoring a stopped release does not re-adopt a stale connection count", a
|
|
|
118
180
|
cwd: "/tmp/rel",
|
|
119
181
|
exitCode: 143,
|
|
120
182
|
exitSignal: "SIGTERM",
|
|
183
|
+
heldProcessGroup: undefined,
|
|
121
184
|
id: "web",
|
|
122
185
|
lastMemoryRestartAt: undefined,
|
|
123
186
|
lastStartReason: "deploy",
|
|
@@ -125,6 +188,7 @@ test("restoring a stopped release does not re-adopt a stale connection count", a
|
|
|
125
188
|
memoryRestarts: 0,
|
|
126
189
|
pid: undefined,
|
|
127
190
|
restarts: 0,
|
|
191
|
+
retirementTimedOut: false,
|
|
128
192
|
rssBytes: undefined,
|
|
129
193
|
startedAt: "2026-10-08T06:22:23.500Z",
|
|
130
194
|
state: "stopped",
|
|
@@ -135,6 +199,7 @@ test("restoring a stopped release does not re-adopt a stale connection count", a
|
|
|
135
199
|
activatedAt: "2026-10-08T06:22:23.583Z",
|
|
136
200
|
connectionCount: 14,
|
|
137
201
|
connections: {http: 14, websocket: 0},
|
|
202
|
+
drain: undefined,
|
|
138
203
|
drainStartedAt: "2026-10-08T07:58:00.464Z",
|
|
139
204
|
ports: {web: 4200},
|
|
140
205
|
processes: [processStatus],
|
|
@@ -150,4 +215,92 @@ test("restoring a stopped release does not re-adopt a stale connection count", a
|
|
|
150
215
|
expect(release.status().connectionCount).toBe(0)
|
|
151
216
|
expect(release.localConnections()).toEqual({http: 0, websocket: 0})
|
|
152
217
|
})
|
|
218
|
+
|
|
219
|
+
test("status reports the drain classification of a draining generation and none of an active one", () => {
|
|
220
|
+
const procPath = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-drain-status-"))
|
|
221
|
+
|
|
222
|
+
try {
|
|
223
|
+
writeProcMember(procPath, 101, 101, "S", 300, 200)
|
|
224
|
+
writeProcMember(procPath, 102, 101, "S", 10, 5)
|
|
225
|
+
|
|
226
|
+
const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}}, () => true, procPath)
|
|
227
|
+
|
|
228
|
+
release.state = "draining"
|
|
229
|
+
release.processes.set("web", drainedProcess(101))
|
|
230
|
+
|
|
231
|
+
const first = release.status().drain
|
|
232
|
+
|
|
233
|
+
expect(first?.status).toBe("unproven")
|
|
234
|
+
expect(first?.sampleWindowMs).toBe(0)
|
|
235
|
+
expect(first?.reason).toBe("first liveness observation; a 20000ms window is required before CPU flatness can be claimed")
|
|
236
|
+
|
|
237
|
+
release.state = "active"
|
|
238
|
+
|
|
239
|
+
expect(release.status().drain).toBe(undefined)
|
|
240
|
+
} finally {
|
|
241
|
+
fs.rmSync(procPath, {force: true, recursive: true})
|
|
242
|
+
}
|
|
243
|
+
})
|
|
244
|
+
|
|
245
|
+
test("a drain observation is reused until a full sample window has passed", () => {
|
|
246
|
+
const procPath = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-drain-window-"))
|
|
247
|
+
|
|
248
|
+
try {
|
|
249
|
+
writeProcMember(procPath, 101, 101, "S", 300, 200)
|
|
250
|
+
writeProcMember(procPath, 102, 101, "S", 10, 5)
|
|
251
|
+
|
|
252
|
+
const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}}, () => true, procPath)
|
|
253
|
+
|
|
254
|
+
release.state = "draining"
|
|
255
|
+
release.processes.set("web", drainedProcess(101))
|
|
256
|
+
|
|
257
|
+
expect(release.status().drain?.status).toBe("unproven")
|
|
258
|
+
|
|
259
|
+
// Backdate the newest observation so the next read is a full window behind it.
|
|
260
|
+
release.drainNewestSample = {
|
|
261
|
+
atMs: Date.now() - 25000,
|
|
262
|
+
processes: [{cpuMs: 5150, heldProcessGroupPgid: undefined, id: "web", liveChildCount: 1, pid: 101, retirementTimedOut: true}]
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
const second = release.status().drain
|
|
266
|
+
|
|
267
|
+
expect(second?.status).toBe("wedged")
|
|
268
|
+
expect(second?.sampleWindowMs).toBe(25000)
|
|
269
|
+
expect(second?.cpuDeltaMs).toBe(0)
|
|
270
|
+
|
|
271
|
+
// The observation taken above is now the newest one. Changing the fixture inside the window
|
|
272
|
+
// must not produce a second observation: the classification still reports the reused sample,
|
|
273
|
+
// which is what keeps a procfs scan off the status path.
|
|
274
|
+
writeProcMember(procPath, 101, 101, "S", 900, 600)
|
|
275
|
+
|
|
276
|
+
const third = release.status().drain
|
|
277
|
+
|
|
278
|
+
expect(third?.status).toBe("wedged")
|
|
279
|
+
expect(third?.sampleWindowMs).toBe(25000)
|
|
280
|
+
expect(third?.cpuDeltaMs).toBe(0)
|
|
281
|
+
} finally {
|
|
282
|
+
fs.rmSync(procPath, {force: true, recursive: true})
|
|
283
|
+
}
|
|
284
|
+
})
|
|
285
|
+
|
|
286
|
+
test("a generation still holding a process group after its tracked process exited is observed through that group", () => {
|
|
287
|
+
const procPath = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-drain-held-"))
|
|
288
|
+
|
|
289
|
+
try {
|
|
290
|
+
// The group leader exited, so only the member that outlived it is left in the group.
|
|
291
|
+
writeProcMember(procPath, 102, 101, "S", 300, 200)
|
|
292
|
+
|
|
293
|
+
const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}}, () => true, procPath)
|
|
294
|
+
|
|
295
|
+
release.state = "draining"
|
|
296
|
+
release.processes.set("web", drainedProcess(undefined, {liveMemberCount: 1, pgid: 101}))
|
|
297
|
+
|
|
298
|
+
const drain = release.status().drain
|
|
299
|
+
|
|
300
|
+
expect(drain?.status).toBe("wedged")
|
|
301
|
+
expect(drain?.reason).toBe("1 live process past its retirement budget with no runner child remaining; 1 tracked process already exited and left web pgid 101 alive")
|
|
302
|
+
} finally {
|
|
303
|
+
fs.rmSync(procPath, {force: true, recursive: true})
|
|
304
|
+
}
|
|
305
|
+
})
|
|
153
306
|
})
|
package/test/rollbridge.test.js
CHANGED
|
@@ -2007,6 +2007,126 @@ test("rollback to a still-draining release stops the old instance instead of orp
|
|
|
2007
2007
|
}
|
|
2008
2008
|
})
|
|
2009
2009
|
|
|
2010
|
+
test("deploy is refused before activation once the draining-generation bound is reached", async () => {
|
|
2011
|
+
const fixture = await createFixture()
|
|
2012
|
+
const config = normalizeConfig({...fixture.config, releaseRetention: {keep: 10, maxAgeMs: 0, maxConcurrentDrainingReleaseGroups: 1}})
|
|
2013
|
+
const daemon = await startDaemon(config)
|
|
2014
|
+
/** @type {WebSocket | undefined} */
|
|
2015
|
+
let socket
|
|
2016
|
+
|
|
2017
|
+
try {
|
|
2018
|
+
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
2019
|
+
|
|
2020
|
+
// A held WebSocket keeps v1 draining after v2 takes traffic over: the measured production
|
|
2021
|
+
// shape where a drain outlasts the deploy cadence.
|
|
2022
|
+
socket = await openWebSocket(daemon)
|
|
2023
|
+
await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
2024
|
+
|
|
2025
|
+
expect(statusRelease(daemon, "v1").state).toBe("draining")
|
|
2026
|
+
|
|
2027
|
+
const beforeRefusal = daemon.status()
|
|
2028
|
+
|
|
2029
|
+
await expect(daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})).rejects.toThrow(/already draining \(v1 progressing \(1 live connection still owned by the generation\)\)\. Activating v3 would make 2, above the releaseRetention\.maxConcurrentDrainingReleaseGroups bound of 1/)
|
|
2030
|
+
|
|
2031
|
+
const afterRefusal = daemon.status()
|
|
2032
|
+
|
|
2033
|
+
// Refusing pre-handoff means nothing was started: no new generation, incumbent intact.
|
|
2034
|
+
expect(afterRefusal.activeReleaseId).toBe("v2")
|
|
2035
|
+
expect(afterRefusal.releases.length).toBe(beforeRefusal.releases.length)
|
|
2036
|
+
expect(afterRefusal.releases.some((release) => release.releaseId === "v3")).toBe(false)
|
|
2037
|
+
expect(await fetchText(daemon, "/release")).toBe("v2")
|
|
2038
|
+
} finally {
|
|
2039
|
+
if (socket) socket.close()
|
|
2040
|
+
await daemon.shutdown()
|
|
2041
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
2042
|
+
}
|
|
2043
|
+
})
|
|
2044
|
+
|
|
2045
|
+
test("a draining-generation bound of 0 leaves concurrent drains unbounded", async () => {
|
|
2046
|
+
const fixture = await createFixture()
|
|
2047
|
+
const config = normalizeConfig({...fixture.config, releaseRetention: {keep: 10, maxAgeMs: 0, maxConcurrentDrainingReleaseGroups: 0}})
|
|
2048
|
+
const daemon = await startDaemon(config)
|
|
2049
|
+
/** @type {WebSocket | undefined} */
|
|
2050
|
+
let socket
|
|
2051
|
+
|
|
2052
|
+
try {
|
|
2053
|
+
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
2054
|
+
socket = await openWebSocket(daemon)
|
|
2055
|
+
await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
2056
|
+
await daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})
|
|
2057
|
+
|
|
2058
|
+
expect(daemon.status().activeReleaseId).toBe("v3")
|
|
2059
|
+
expect(statusRelease(daemon, "v1").state).toBe("draining")
|
|
2060
|
+
} finally {
|
|
2061
|
+
if (socket) socket.close()
|
|
2062
|
+
await daemon.shutdown()
|
|
2063
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
2064
|
+
}
|
|
2065
|
+
})
|
|
2066
|
+
|
|
2067
|
+
test("status reports the drain classification of each draining generation", async () => {
|
|
2068
|
+
const fixture = await createFixture()
|
|
2069
|
+
const daemon = await startDaemon(fixture.config)
|
|
2070
|
+
/** @type {WebSocket | undefined} */
|
|
2071
|
+
let socket
|
|
2072
|
+
|
|
2073
|
+
try {
|
|
2074
|
+
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
2075
|
+
socket = await openWebSocket(daemon)
|
|
2076
|
+
await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
2077
|
+
|
|
2078
|
+
const draining = statusRelease(daemon, "v1")
|
|
2079
|
+
|
|
2080
|
+
expect(draining.state).toBe("draining")
|
|
2081
|
+
expect(draining.drain?.status).toBe("progressing")
|
|
2082
|
+
expect(draining.drain?.reason).toBe("1 live connection still owned by the generation")
|
|
2083
|
+
expect(statusRelease(daemon, "v2").drain).toBe(undefined)
|
|
2084
|
+
} finally {
|
|
2085
|
+
if (socket) socket.close()
|
|
2086
|
+
await daemon.shutdown()
|
|
2087
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
2088
|
+
}
|
|
2089
|
+
})
|
|
2090
|
+
|
|
2091
|
+
test("rollback refuses to restore a drained generation that still holds live processes", async () => {
|
|
2092
|
+
const fixture = await createFixture({nonBlockingDrainWorker: true, workerStopDelayMs: 6000})
|
|
2093
|
+
const config = normalizeConfig({
|
|
2094
|
+
...fixture.config,
|
|
2095
|
+
processes: fixture.config.processes.map((processConfig) => processConfig.id === "worker"
|
|
2096
|
+
? {...processConfig, gracefulStopMs: "indefinite"}
|
|
2097
|
+
: processConfig),
|
|
2098
|
+
proxy: {...fixture.config.proxy, nonBlockingDrainTimeoutMs: 200}
|
|
2099
|
+
})
|
|
2100
|
+
const daemon = await startDaemon(config)
|
|
2101
|
+
|
|
2102
|
+
try {
|
|
2103
|
+
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
2104
|
+
await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
2105
|
+
|
|
2106
|
+
// The retirement budget expires, Rollbridge records the generation as drained, and the
|
|
2107
|
+
// worker stays alive with its in-flight work — the shape a restore must not re-deploy.
|
|
2108
|
+
// The tracked process (the shell wrapper) is already gone, so the only honest record of
|
|
2109
|
+
// the leak is the process group that outlived it.
|
|
2110
|
+
await waitFor(() => statusRelease(daemon, "v1").state === "stopped")
|
|
2111
|
+
|
|
2112
|
+
const worker = statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")
|
|
2113
|
+
|
|
2114
|
+
if (!worker?.heldProcessGroup) {
|
|
2115
|
+
throw new Error(`the drained generation should still hold the worker's process group, got ${JSON.stringify(worker && {pid: worker.pid, retirementTimedOut: worker.retirementTimedOut, state: worker.state})}`)
|
|
2116
|
+
}
|
|
2117
|
+
|
|
2118
|
+
expect(worker.pid).toBeUndefined()
|
|
2119
|
+
expect(worker.retirementTimedOut).toBe(true)
|
|
2120
|
+
expect(worker.state).toBe("stopped")
|
|
2121
|
+
await expect(daemon.rollback({releaseId: "v1"})).rejects.toThrow(/recorded as drained while 1 process of that generation still hold live work \(worker: process group \d+ with \d+ live member\(s\) still alive after the tracked process exited\)\. Restoring it would re-register those processes under a new provenance and fail with a guardian provenance mismatch\. The guardian's active release is "v2"/)
|
|
2122
|
+
expect(daemon.status().activeReleaseId).toBe("v2")
|
|
2123
|
+
expect(await fetchText(daemon, "/release")).toBe("v2")
|
|
2124
|
+
} finally {
|
|
2125
|
+
await daemon.shutdown()
|
|
2126
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
2127
|
+
}
|
|
2128
|
+
})
|
|
2129
|
+
|
|
2010
2130
|
test("the rollback control command switches traffic over the socket", async () => {
|
|
2011
2131
|
const fixture = await createFixture()
|
|
2012
2132
|
const daemon = await startDaemon(fixture.config)
|