rollbridge 0.1.63 → 0.1.66
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.d/20261115-bound-draining-release-groups.md +62 -0
- package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +27 -0
- package/docs/cli.md +59 -0
- package/docs/config.md +26 -1
- package/package.json +1 -1
- package/src/config.js +13 -2
- package/src/daemon.js +146 -1
- package/src/drain-policy.js +159 -0
- package/src/managed-process.js +99 -3
- package/src/process-guardian.js +4 -5
- package/src/process-memory.js +62 -2
- package/src/release-group.js +93 -2
- package/test/config-validation.test.js +11 -4
- package/test/drain-policy.test.js +199 -0
- package/test/managed-process.test.js +11 -1
- package/test/process-memory.test.js +47 -1
- package/test/release-group.test.js +127 -2
- package/test/rollbridge.test.js +193 -0
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
import {describe, expect, test} from "@velocious/testing"
|
|
4
|
+
import {admitGenerationRelease, classifyDrainProgress} from "../src/drain-policy.js"
|
|
5
|
+
|
|
6
|
+
describe("drain-policy", () => {
|
|
7
|
+
|
|
8
|
+
const WINDOW_MS = 20000
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* @param {object} [options] - Observation shape.
|
|
12
|
+
* @param {number} [options.atMs] - Observation time.
|
|
13
|
+
* @param {number | undefined} [options.cpuMs] - Cumulative CPU of the group.
|
|
14
|
+
* @param {number | undefined} [options.heldProcessGroupPgid] - Group still alive after its tracked process exited.
|
|
15
|
+
* @param {number} [options.liveChildCount] - Runner children other than the group leader.
|
|
16
|
+
* @param {number | undefined} [options.pid] - Live pid, or undefined when the process is gone.
|
|
17
|
+
* @param {boolean} [options.retirementTimedOut] - Whether the supervisor already gave up on it.
|
|
18
|
+
* @returns {import("../src/drain-policy.js").DrainLivenessSample} One liveness observation.
|
|
19
|
+
*/
|
|
20
|
+
function sample(options = {}) {
|
|
21
|
+
return {
|
|
22
|
+
atMs: options.atMs ?? 1000000,
|
|
23
|
+
processes: [{
|
|
24
|
+
cpuMs: "cpuMs" in options ? options.cpuMs : 5000,
|
|
25
|
+
heldProcessGroupPgid: options.heldProcessGroupPgid,
|
|
26
|
+
id: "worker",
|
|
27
|
+
liveChildCount: options.liveChildCount ?? 1,
|
|
28
|
+
pid: "pid" in options ? options.pid : 4242,
|
|
29
|
+
retirementTimedOut: options.retirementTimedOut ?? true
|
|
30
|
+
}]
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* @param {object} [options] - Classification inputs.
|
|
36
|
+
* @param {number} [options.connectionCount] - Live connections owned by the generation.
|
|
37
|
+
* @param {import("../src/drain-policy.js").DrainLivenessSample} [options.current] - Newest observation.
|
|
38
|
+
* @param {import("../src/drain-policy.js").DrainLivenessSample | undefined} [options.previous] - Older observation.
|
|
39
|
+
* @returns {import("../src/drain-policy.js").DrainClassification} Classification.
|
|
40
|
+
*/
|
|
41
|
+
function classify(options = {}) {
|
|
42
|
+
return classifyDrainProgress({
|
|
43
|
+
connectionCount: options.connectionCount ?? 0,
|
|
44
|
+
currentSample: options.current ?? sample(),
|
|
45
|
+
minSampleWindowMs: WINDOW_MS,
|
|
46
|
+
previousSample: options.previous
|
|
47
|
+
})
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
test("a generation that still owns connections is draining, never wedged", () => {
|
|
51
|
+
const result = classify({connectionCount: 3, current: sample({liveChildCount: 0})})
|
|
52
|
+
|
|
53
|
+
expect(result.status).toBe("progressing")
|
|
54
|
+
expect(result.reason).toBe("3 live connections still owned by the generation")
|
|
55
|
+
})
|
|
56
|
+
|
|
57
|
+
test("a process still inside its retirement budget is draining", () => {
|
|
58
|
+
const result = classify({current: sample({liveChildCount: 0, retirementTimedOut: false})})
|
|
59
|
+
|
|
60
|
+
expect(result.status).toBe("progressing")
|
|
61
|
+
expect(result.reason).toBe("process worker is still inside its retirement budget")
|
|
62
|
+
})
|
|
63
|
+
|
|
64
|
+
test("a live process past its retirement budget with no runner child left is wedged", () => {
|
|
65
|
+
const result = classify({current: sample({liveChildCount: 0})})
|
|
66
|
+
|
|
67
|
+
expect(result.status).toBe("wedged")
|
|
68
|
+
expect(result.reason).toBe("1 live process past its retirement budget with no runner child remaining")
|
|
69
|
+
})
|
|
70
|
+
|
|
71
|
+
test("flat CPU across a sufficient window is wedged, and the window is reported", () => {
|
|
72
|
+
const previous = sample({atMs: 1000000, cpuMs: 5000})
|
|
73
|
+
const current = sample({atMs: 1025000, cpuMs: 5000, liveChildCount: 3})
|
|
74
|
+
const result = classify({current, previous})
|
|
75
|
+
|
|
76
|
+
expect(result.status).toBe("wedged")
|
|
77
|
+
expect(result.sampleWindowMs).toBe(25000)
|
|
78
|
+
expect(result.cpuDeltaMs).toBe(0)
|
|
79
|
+
expect(result.reason).toBe("1 live process with 3 runner children, no live connection, every retirement budget expired, CPU flat across 25000ms")
|
|
80
|
+
})
|
|
81
|
+
|
|
82
|
+
test("CPU that advanced across the window is a progressing drain", () => {
|
|
83
|
+
const previous = sample({atMs: 1000000, cpuMs: 5000})
|
|
84
|
+
const current = sample({atMs: 1025000, cpuMs: 9000})
|
|
85
|
+
const result = classify({current, previous})
|
|
86
|
+
|
|
87
|
+
expect(result.status).toBe("progressing")
|
|
88
|
+
expect(result.cpuDeltaMs).toBe(4000)
|
|
89
|
+
expect(result.reason).toBe("CPU advanced 4000ms across 25000ms")
|
|
90
|
+
})
|
|
91
|
+
|
|
92
|
+
test("CPU leaving the group with an exited member is a progressing drain", () => {
|
|
93
|
+
const previous = sample({atMs: 1000000, cpuMs: 9000})
|
|
94
|
+
const current = sample({atMs: 1025000, cpuMs: 5000})
|
|
95
|
+
const result = classify({current, previous})
|
|
96
|
+
|
|
97
|
+
expect(result.status).toBe("progressing")
|
|
98
|
+
expect(result.cpuDeltaMs).toBe(-4000)
|
|
99
|
+
expect(result.reason).toBe("the group changed (1 live process, -4000ms of consumed CPU left the group) across 25000ms")
|
|
100
|
+
})
|
|
101
|
+
|
|
102
|
+
test("the first observation cannot claim flatness", () => {
|
|
103
|
+
const result = classify({current: sample()})
|
|
104
|
+
|
|
105
|
+
expect(result.status).toBe("unproven")
|
|
106
|
+
expect(result.sampleWindowMs).toBe(0)
|
|
107
|
+
expect(result.reason).toBe("first liveness observation; a 20000ms window is required before CPU flatness can be claimed")
|
|
108
|
+
})
|
|
109
|
+
|
|
110
|
+
test("a window shorter than the minimum cannot claim flatness", () => {
|
|
111
|
+
const previous = sample({atMs: 1000000})
|
|
112
|
+
const current = sample({atMs: 1010000})
|
|
113
|
+
const result = classify({current, previous})
|
|
114
|
+
|
|
115
|
+
expect(result.status).toBe("unproven")
|
|
116
|
+
expect(result.sampleWindowMs).toBe(10000)
|
|
117
|
+
expect(result.reason).toBe("liveness window 10000ms is below the 20000ms minimum")
|
|
118
|
+
})
|
|
119
|
+
|
|
120
|
+
test("an unmeasurable CPU counter is unproven, never wedged", () => {
|
|
121
|
+
const previous = sample({atMs: 1000000, cpuMs: undefined})
|
|
122
|
+
const current = sample({atMs: 1025000, cpuMs: undefined})
|
|
123
|
+
const result = classify({current, previous})
|
|
124
|
+
|
|
125
|
+
expect(result.status).toBe("unproven")
|
|
126
|
+
expect(result.cpuDeltaMs).toBe(undefined)
|
|
127
|
+
expect(result.reason).toBe("process CPU is not measurable on this host")
|
|
128
|
+
})
|
|
129
|
+
|
|
130
|
+
test("a generation with no live process left is not a wedge", () => {
|
|
131
|
+
const result = classify({current: sample({pid: undefined})})
|
|
132
|
+
|
|
133
|
+
expect(result.status).toBe("progressing")
|
|
134
|
+
expect(result.reason).toBe("no live process remains in the generation")
|
|
135
|
+
})
|
|
136
|
+
|
|
137
|
+
test("a process group that outlived its tracked process is still a live process of the generation", () => {
|
|
138
|
+
const result = classify({current: sample({heldProcessGroupPgid: 4242, liveChildCount: 0, pid: undefined})})
|
|
139
|
+
|
|
140
|
+
expect(result.status).toBe("wedged")
|
|
141
|
+
expect(result.reason).toBe("1 live process past its retirement budget with no runner child remaining; 1 tracked process already exited and left worker pgid 4242 alive")
|
|
142
|
+
})
|
|
143
|
+
|
|
144
|
+
test("a held process group that still consumes CPU across the window is a progressing drain", () => {
|
|
145
|
+
const previous = sample({atMs: 1000000, cpuMs: 5000, heldProcessGroupPgid: 4242, liveChildCount: 2, pid: undefined})
|
|
146
|
+
const current = sample({atMs: 1025000, cpuMs: 9000, heldProcessGroupPgid: 4242, liveChildCount: 2, pid: undefined})
|
|
147
|
+
const result = classify({current, previous})
|
|
148
|
+
|
|
149
|
+
expect(result.status).toBe("progressing")
|
|
150
|
+
expect(result.cpuDeltaMs).toBe(4000)
|
|
151
|
+
expect(result.reason).toBe("CPU advanced 4000ms across 25000ms")
|
|
152
|
+
})
|
|
153
|
+
|
|
154
|
+
test("admission allows a handoff that stays within the bound and counts the incumbent", () => {
|
|
155
|
+
const admission = admitGenerationRelease({
|
|
156
|
+
candidateReleaseId: "v3",
|
|
157
|
+
draining: [{reason: "1 live connection still owned by the generation", releaseId: "v1", status: "progressing"}],
|
|
158
|
+
incumbentWillDrain: true,
|
|
159
|
+
maxConcurrentDrainingReleaseGroups: 2
|
|
160
|
+
})
|
|
161
|
+
|
|
162
|
+
expect(admission.allowed).toBe(true)
|
|
163
|
+
expect(admission.wouldBeDrainingCount).toBe(2)
|
|
164
|
+
expect(admission.reason).toBe(undefined)
|
|
165
|
+
})
|
|
166
|
+
|
|
167
|
+
test("admission refuses before activation and names every draining group with its classification", () => {
|
|
168
|
+
const admission = admitGenerationRelease({
|
|
169
|
+
candidateReleaseId: "20261115073717",
|
|
170
|
+
draining: [
|
|
171
|
+
{reason: "1 live process with 3 runner children, no live connection, every retirement budget expired, CPU flat across 25000ms", releaseId: "20261115063805", status: "wedged"},
|
|
172
|
+
{reason: "2 live connections still owned by the generation", releaseId: "20261114232900", status: "progressing"}
|
|
173
|
+
],
|
|
174
|
+
incumbentWillDrain: true,
|
|
175
|
+
maxConcurrentDrainingReleaseGroups: 2
|
|
176
|
+
})
|
|
177
|
+
|
|
178
|
+
expect(admission.allowed).toBe(false)
|
|
179
|
+
expect(admission.wouldBeDrainingCount).toBe(3)
|
|
180
|
+
expect(admission.reason).toContain("Deploy refused before activation: 2 release groups already draining")
|
|
181
|
+
expect(admission.reason).toContain("20261115063805 wedged")
|
|
182
|
+
expect(admission.reason).toContain("Activating 20261115073717 would make 3, above the releaseRetention.maxConcurrentDrainingReleaseGroups bound of 2")
|
|
183
|
+
})
|
|
184
|
+
|
|
185
|
+
test("a first deploy with no incumbent adds no draining group", () => {
|
|
186
|
+
const admission = admitGenerationRelease({candidateReleaseId: "v1", draining: [], incumbentWillDrain: false, maxConcurrentDrainingReleaseGroups: 1})
|
|
187
|
+
|
|
188
|
+
expect(admission.allowed).toBe(true)
|
|
189
|
+
expect(admission.wouldBeDrainingCount).toBe(0)
|
|
190
|
+
})
|
|
191
|
+
|
|
192
|
+
test("a bound of 0 disables the admission check", () => {
|
|
193
|
+
const draining = Array.from({length: 30}, (_, index) => ({reason: "no runner child remaining", releaseId: `v${index}`, status: /** @type {const} */ ("wedged")}))
|
|
194
|
+
const admission = admitGenerationRelease({candidateReleaseId: "v31", draining, incumbentWillDrain: true, maxConcurrentDrainingReleaseGroups: 0})
|
|
195
|
+
|
|
196
|
+
expect(admission.allowed).toBe(true)
|
|
197
|
+
expect(admission.wouldBeDrainingCount).toBe(31)
|
|
198
|
+
})
|
|
199
|
+
})
|
|
@@ -5,7 +5,7 @@ import os from "node:os"
|
|
|
5
5
|
import path from "node:path"
|
|
6
6
|
import {describe, expect, test} from "@velocious/testing"
|
|
7
7
|
import {fileURLToPath} from "node:url"
|
|
8
|
-
import ManagedProcess from "../src/managed-process.js"
|
|
8
|
+
import ManagedProcess, {childRecordIsGone} from "../src/managed-process.js"
|
|
9
9
|
|
|
10
10
|
describe("managed-process", () => {
|
|
11
11
|
|
|
@@ -930,4 +930,14 @@ test("the unlimited constant-delay fast path still applies maxDelayMs", () => {
|
|
|
930
930
|
|
|
931
931
|
expect(queued).toBe(5)
|
|
932
932
|
})
|
|
933
|
+
|
|
934
|
+
test("the guardian fence releases only a record that is terminal and holds no process group", () => {
|
|
935
|
+
expect(childRecordIsGone({heldProcessGroup: undefined, pid: undefined, state: "stopped"})).toBe(true)
|
|
936
|
+
expect(childRecordIsGone({heldProcessGroup: undefined, pid: undefined, state: "failed"})).toBe(true)
|
|
937
|
+
expect(childRecordIsGone({heldProcessGroup: undefined, pid: 4242, state: "running"})).toBe(false)
|
|
938
|
+
|
|
939
|
+
// The shape a drained generation leaves behind: the tracked process record is terminal, but the
|
|
940
|
+
// group it led still holds live members, so the key is still held by that generation.
|
|
941
|
+
expect(childRecordIsGone({heldProcessGroup: {liveMemberCount: 1, pgid: 4242}, pid: undefined, state: "stopped"})).toBe(false)
|
|
942
|
+
})
|
|
933
943
|
})
|
|
@@ -4,7 +4,7 @@ import fs from "node:fs"
|
|
|
4
4
|
import {describe, expect, test} from "@velocious/testing"
|
|
5
5
|
import os from "node:os"
|
|
6
6
|
import path from "node:path"
|
|
7
|
-
import {measureProcessGroupRssBytes, processGroupHasLiveMembers, processGroupMembers} from "../src/process-memory.js"
|
|
7
|
+
import {measureProcessGroupRssBytes, processGroupHasLiveMembers, processGroupMembers, sampleProcessGroup} from "../src/process-memory.js"
|
|
8
8
|
|
|
9
9
|
describe("process-memory", () => {
|
|
10
10
|
|
|
@@ -59,4 +59,50 @@ test("treats a process group containing only defunct members as stopped", () =>
|
|
|
59
59
|
fs.rmSync(procPath, {force: true, recursive: true})
|
|
60
60
|
}
|
|
61
61
|
})
|
|
62
|
+
|
|
63
|
+
test("samples a process group's cumulative CPU and its live member count", () => {
|
|
64
|
+
const procPath = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-proc-cpu-"))
|
|
65
|
+
|
|
66
|
+
try {
|
|
67
|
+
// utime and stime are stat fields 14 and 15: index 11 and 12 after the parenthesised comm,
|
|
68
|
+
// so exactly eight placeholder fields sit between pgrp and utime.
|
|
69
|
+
fs.mkdirSync(path.join(procPath, "101"))
|
|
70
|
+
fs.writeFileSync(path.join(procPath, "101", "stat"), "101 (worker) S 1 77 0 0 0 0 0 0 0 0 300 200")
|
|
71
|
+
fs.mkdirSync(path.join(procPath, "102"))
|
|
72
|
+
fs.writeFileSync(path.join(procPath, "102", "stat"), "102 (runner) Z 1 77 0 0 0 0 0 0 0 0 10 5")
|
|
73
|
+
fs.mkdirSync(path.join(procPath, "103"))
|
|
74
|
+
fs.writeFileSync(path.join(procPath, "103", "stat"), "103 (other) S 1 99 0 0 0 0 0 0 0 0 9000 9000")
|
|
75
|
+
|
|
76
|
+
const sampled = sampleProcessGroup(77, procPath)
|
|
77
|
+
|
|
78
|
+
// Defunct members keep their consumed CPU, so the total covers both group members and
|
|
79
|
+
// excludes the foreign group 99 entirely: (300 + 200 + 10 + 5) ticks * 10ms per tick.
|
|
80
|
+
expect(sampled).toEqual({cpuMs: 5150, liveMemberCount: 1, memberCount: 2})
|
|
81
|
+
} finally {
|
|
82
|
+
fs.rmSync(procPath, {force: true, recursive: true})
|
|
83
|
+
}
|
|
84
|
+
})
|
|
85
|
+
|
|
86
|
+
test("reports group counts when a member's CPU fields are unreadable", () => {
|
|
87
|
+
const procPath = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-proc-cpu-partial-"))
|
|
88
|
+
|
|
89
|
+
try {
|
|
90
|
+
fs.mkdirSync(path.join(procPath, "101"))
|
|
91
|
+
fs.writeFileSync(path.join(procPath, "101", "stat"), "101 (worker) S 1 77 0 0")
|
|
92
|
+
|
|
93
|
+
expect(sampleProcessGroup(77, procPath)).toEqual({cpuMs: undefined, liveMemberCount: 1, memberCount: 1})
|
|
94
|
+
} finally {
|
|
95
|
+
fs.rmSync(procPath, {force: true, recursive: true})
|
|
96
|
+
}
|
|
97
|
+
})
|
|
98
|
+
|
|
99
|
+
test("reports a zero sample for a process group with no members", () => {
|
|
100
|
+
expect(sampleProcessGroup(2147483646)).toEqual({cpuMs: 0, liveMemberCount: 0, memberCount: 0})
|
|
101
|
+
})
|
|
102
|
+
|
|
103
|
+
linuxTest("samples the current process group's CPU as a positive measurable total", () => {
|
|
104
|
+
const sampled = sampleProcessGroup(currentProcessGroupId())
|
|
105
|
+
|
|
106
|
+
expect({value: Boolean(sampled && typeof sampled.cpuMs === "number" && sampled.cpuMs >= 0 && sampled.liveMemberCount >= 1), context: `expected a sampled group, got ${JSON.stringify(sampled)}`}).toMatchObject({value: true})
|
|
107
|
+
})
|
|
62
108
|
})
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
3
|
import {describe, expect, test} from "@velocious/testing"
|
|
4
|
+
import fs from "node:fs"
|
|
5
|
+
import os from "node:os"
|
|
6
|
+
import path from "node:path"
|
|
4
7
|
import ReleaseGroup from "../src/release-group.js"
|
|
5
8
|
import {normalizeConfig} from "../src/config.js"
|
|
6
9
|
|
|
@@ -9,9 +12,10 @@ describe("release-group", () => {
|
|
|
9
12
|
/**
|
|
10
13
|
* @param {import("../src/json.js").JsonValue} webProcess - The single proxied process definition.
|
|
11
14
|
* @param {() => boolean} [shouldStart] - Whether process starts remain allowed.
|
|
15
|
+
* @param {string} [procPath] - Procfs root for drain liveness observations.
|
|
12
16
|
* @returns {ReleaseGroup} A release group ready for buildProcess.
|
|
13
17
|
*/
|
|
14
|
-
function buildRelease(webProcess, shouldStart = () => true) {
|
|
18
|
+
function buildRelease(webProcess, shouldStart = () => true, procPath = "/proc") {
|
|
15
19
|
const config = normalizeConfig({
|
|
16
20
|
application: "demo",
|
|
17
21
|
control: {path: "/tmp/rollbridge-release-group.sock"},
|
|
@@ -19,7 +23,37 @@ function buildRelease(webProcess, shouldStart = () => true) {
|
|
|
19
23
|
proxy: {host: "127.0.0.1", port: 0}
|
|
20
24
|
})
|
|
21
25
|
|
|
22
|
-
return new ReleaseGroup({config, logger: () => {}, releaseId: "v1", releasePath: "/tmp/rel", revision: "v1", shouldStart})
|
|
26
|
+
return new ReleaseGroup({config, logger: () => {}, procPath, releaseId: "v1", releasePath: "/tmp/rel", revision: "v1", shouldStart})
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* @param {string} procPath - Procfs root.
|
|
31
|
+
* @param {number} pid - Process id.
|
|
32
|
+
* @param {number} pgrp - Process group id.
|
|
33
|
+
* @param {string} state - Process state letter.
|
|
34
|
+
* @param {number} utimeTicks - Cumulative user CPU in clock ticks.
|
|
35
|
+
* @param {number} stimeTicks - Cumulative system CPU in clock ticks.
|
|
36
|
+
* @returns {void} One synthetic procfs member.
|
|
37
|
+
*/
|
|
38
|
+
function writeProcMember(procPath, pid, pgrp, state, utimeTicks, stimeTicks) {
|
|
39
|
+
fs.mkdirSync(path.join(procPath, String(pid)), {recursive: true})
|
|
40
|
+
|
|
41
|
+
// Eight placeholder fields sit between pgrp and utime, which is stat field 14 (index 11
|
|
42
|
+
// after the parenthesised comm); stime is field 15 (index 12).
|
|
43
|
+
fs.writeFileSync(path.join(procPath, String(pid), "stat"), `${pid} (proc) ${state} 1 ${pgrp} 0 0 0 0 0 0 0 0 ${utimeTicks} ${stimeTicks}`)
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* @param {number | undefined} pid - Live pid, or undefined when the tracked process is gone.
|
|
48
|
+
* @param {import("../src/managed-process.js").HeldProcessGroup | undefined} [heldProcessGroup] - Group still alive after the tracked process exited.
|
|
49
|
+
* @returns {import("../src/managed-process.js").default} A process whose record is already terminal.
|
|
50
|
+
*/
|
|
51
|
+
function drainedProcess(pid, heldProcessGroup = undefined) {
|
|
52
|
+
// eslint-disable-next-line local/no-unknown-jsdoc-type
|
|
53
|
+
return /** @type {import("../src/managed-process.js").default} */ (/** @type {unknown} */ ({
|
|
54
|
+
retirementTimedOut: true,
|
|
55
|
+
status: () => ({heldProcessGroup, pid, retirementTimedOut: true, state: "stopped"})
|
|
56
|
+
}))
|
|
23
57
|
}
|
|
24
58
|
|
|
25
59
|
test("templates interpolate values from the daemon environment", () => {
|
|
@@ -146,6 +180,7 @@ test("restoring a stopped release does not re-adopt a stale connection count", a
|
|
|
146
180
|
cwd: "/tmp/rel",
|
|
147
181
|
exitCode: 143,
|
|
148
182
|
exitSignal: "SIGTERM",
|
|
183
|
+
heldProcessGroup: undefined,
|
|
149
184
|
id: "web",
|
|
150
185
|
lastMemoryRestartAt: undefined,
|
|
151
186
|
lastStartReason: "deploy",
|
|
@@ -153,6 +188,7 @@ test("restoring a stopped release does not re-adopt a stale connection count", a
|
|
|
153
188
|
memoryRestarts: 0,
|
|
154
189
|
pid: undefined,
|
|
155
190
|
restarts: 0,
|
|
191
|
+
retirementTimedOut: false,
|
|
156
192
|
rssBytes: undefined,
|
|
157
193
|
startedAt: "2026-10-08T06:22:23.500Z",
|
|
158
194
|
state: "stopped",
|
|
@@ -163,6 +199,7 @@ test("restoring a stopped release does not re-adopt a stale connection count", a
|
|
|
163
199
|
activatedAt: "2026-10-08T06:22:23.583Z",
|
|
164
200
|
connectionCount: 14,
|
|
165
201
|
connections: {http: 14, websocket: 0},
|
|
202
|
+
drain: undefined,
|
|
166
203
|
drainStartedAt: "2026-10-08T07:58:00.464Z",
|
|
167
204
|
ports: {web: 4200},
|
|
168
205
|
processes: [processStatus],
|
|
@@ -178,4 +215,92 @@ test("restoring a stopped release does not re-adopt a stale connection count", a
|
|
|
178
215
|
expect(release.status().connectionCount).toBe(0)
|
|
179
216
|
expect(release.localConnections()).toEqual({http: 0, websocket: 0})
|
|
180
217
|
})
|
|
218
|
+
|
|
219
|
+
test("status reports the drain classification of a draining generation and none of an active one", () => {
|
|
220
|
+
const procPath = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-drain-status-"))
|
|
221
|
+
|
|
222
|
+
try {
|
|
223
|
+
writeProcMember(procPath, 101, 101, "S", 300, 200)
|
|
224
|
+
writeProcMember(procPath, 102, 101, "S", 10, 5)
|
|
225
|
+
|
|
226
|
+
const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}}, () => true, procPath)
|
|
227
|
+
|
|
228
|
+
release.state = "draining"
|
|
229
|
+
release.processes.set("web", drainedProcess(101))
|
|
230
|
+
|
|
231
|
+
const first = release.status().drain
|
|
232
|
+
|
|
233
|
+
expect(first?.status).toBe("unproven")
|
|
234
|
+
expect(first?.sampleWindowMs).toBe(0)
|
|
235
|
+
expect(first?.reason).toBe("first liveness observation; a 20000ms window is required before CPU flatness can be claimed")
|
|
236
|
+
|
|
237
|
+
release.state = "active"
|
|
238
|
+
|
|
239
|
+
expect(release.status().drain).toBe(undefined)
|
|
240
|
+
} finally {
|
|
241
|
+
fs.rmSync(procPath, {force: true, recursive: true})
|
|
242
|
+
}
|
|
243
|
+
})
|
|
244
|
+
|
|
245
|
+
test("a drain observation is reused until a full sample window has passed", () => {
|
|
246
|
+
const procPath = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-drain-window-"))
|
|
247
|
+
|
|
248
|
+
try {
|
|
249
|
+
writeProcMember(procPath, 101, 101, "S", 300, 200)
|
|
250
|
+
writeProcMember(procPath, 102, 101, "S", 10, 5)
|
|
251
|
+
|
|
252
|
+
const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}}, () => true, procPath)
|
|
253
|
+
|
|
254
|
+
release.state = "draining"
|
|
255
|
+
release.processes.set("web", drainedProcess(101))
|
|
256
|
+
|
|
257
|
+
expect(release.status().drain?.status).toBe("unproven")
|
|
258
|
+
|
|
259
|
+
// Backdate the newest observation so the next read is a full window behind it.
|
|
260
|
+
release.drainNewestSample = {
|
|
261
|
+
atMs: Date.now() - 25000,
|
|
262
|
+
processes: [{cpuMs: 5150, heldProcessGroupPgid: undefined, id: "web", liveChildCount: 1, pid: 101, retirementTimedOut: true}]
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
const second = release.status().drain
|
|
266
|
+
|
|
267
|
+
expect(second?.status).toBe("wedged")
|
|
268
|
+
expect(second?.sampleWindowMs).toBe(25000)
|
|
269
|
+
expect(second?.cpuDeltaMs).toBe(0)
|
|
270
|
+
|
|
271
|
+
// The observation taken above is now the newest one. Changing the fixture inside the window
|
|
272
|
+
// must not produce a second observation: the classification still reports the reused sample,
|
|
273
|
+
// which is what keeps a procfs scan off the status path.
|
|
274
|
+
writeProcMember(procPath, 101, 101, "S", 900, 600)
|
|
275
|
+
|
|
276
|
+
const third = release.status().drain
|
|
277
|
+
|
|
278
|
+
expect(third?.status).toBe("wedged")
|
|
279
|
+
expect(third?.sampleWindowMs).toBe(25000)
|
|
280
|
+
expect(third?.cpuDeltaMs).toBe(0)
|
|
281
|
+
} finally {
|
|
282
|
+
fs.rmSync(procPath, {force: true, recursive: true})
|
|
283
|
+
}
|
|
284
|
+
})
|
|
285
|
+
|
|
286
|
+
test("a generation still holding a process group after its tracked process exited is observed through that group", () => {
|
|
287
|
+
const procPath = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-drain-held-"))
|
|
288
|
+
|
|
289
|
+
try {
|
|
290
|
+
// The group leader exited, so only the member that outlived it is left in the group.
|
|
291
|
+
writeProcMember(procPath, 102, 101, "S", 300, 200)
|
|
292
|
+
|
|
293
|
+
const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}}, () => true, procPath)
|
|
294
|
+
|
|
295
|
+
release.state = "draining"
|
|
296
|
+
release.processes.set("web", drainedProcess(undefined, {liveMemberCount: 1, pgid: 101}))
|
|
297
|
+
|
|
298
|
+
const drain = release.status().drain
|
|
299
|
+
|
|
300
|
+
expect(drain?.status).toBe("wedged")
|
|
301
|
+
expect(drain?.reason).toBe("1 live process past its retirement budget with no runner child remaining; 1 tracked process already exited and left web pgid 101 alive")
|
|
302
|
+
} finally {
|
|
303
|
+
fs.rmSync(procPath, {force: true, recursive: true})
|
|
304
|
+
}
|
|
305
|
+
})
|
|
181
306
|
})
|
package/test/rollbridge.test.js
CHANGED
|
@@ -2007,6 +2007,199 @@ test("rollback to a still-draining release stops the old instance instead of orp
|
|
|
2007
2007
|
}
|
|
2008
2008
|
})
|
|
2009
2009
|
|
|
2010
|
+
test("deploy is refused before activation once the draining-generation bound is reached", async () => {
|
|
2011
|
+
const fixture = await createFixture()
|
|
2012
|
+
const config = normalizeConfig({...fixture.config, releaseRetention: {keep: 10, maxAgeMs: 0, maxConcurrentDrainingReleaseGroups: 1}})
|
|
2013
|
+
const daemon = await startDaemon(config)
|
|
2014
|
+
/** @type {WebSocket | undefined} */
|
|
2015
|
+
let socket
|
|
2016
|
+
|
|
2017
|
+
try {
|
|
2018
|
+
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
2019
|
+
|
|
2020
|
+
// A held WebSocket keeps v1 draining after v2 takes traffic over: the measured production
|
|
2021
|
+
// shape where a drain outlasts the deploy cadence.
|
|
2022
|
+
socket = await openWebSocket(daemon)
|
|
2023
|
+
await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
2024
|
+
|
|
2025
|
+
expect(statusRelease(daemon, "v1").state).toBe("draining")
|
|
2026
|
+
|
|
2027
|
+
const beforeRefusal = daemon.status()
|
|
2028
|
+
|
|
2029
|
+
await expect(daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})).rejects.toThrow(/already draining \(v1 progressing \(1 live connection still owned by the generation\)\)\. Activating v3 would make 2, above the releaseRetention\.maxConcurrentDrainingReleaseGroups bound of 1/)
|
|
2030
|
+
|
|
2031
|
+
const afterRefusal = daemon.status()
|
|
2032
|
+
|
|
2033
|
+
// Refusing pre-handoff means nothing was started: no new generation, incumbent intact.
|
|
2034
|
+
expect(afterRefusal.activeReleaseId).toBe("v2")
|
|
2035
|
+
expect(afterRefusal.releases.length).toBe(beforeRefusal.releases.length)
|
|
2036
|
+
expect(afterRefusal.releases.some((release) => release.releaseId === "v3")).toBe(false)
|
|
2037
|
+
expect(await fetchText(daemon, "/release")).toBe("v2")
|
|
2038
|
+
} finally {
|
|
2039
|
+
if (socket) socket.close()
|
|
2040
|
+
await daemon.shutdown()
|
|
2041
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
2042
|
+
}
|
|
2043
|
+
})
|
|
2044
|
+
|
|
2045
|
+
test("a draining-generation bound of 0 leaves concurrent drains unbounded", async () => {
|
|
2046
|
+
const fixture = await createFixture()
|
|
2047
|
+
const config = normalizeConfig({...fixture.config, releaseRetention: {keep: 10, maxAgeMs: 0, maxConcurrentDrainingReleaseGroups: 0}})
|
|
2048
|
+
const daemon = await startDaemon(config)
|
|
2049
|
+
/** @type {WebSocket | undefined} */
|
|
2050
|
+
let socket
|
|
2051
|
+
|
|
2052
|
+
try {
|
|
2053
|
+
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
2054
|
+
socket = await openWebSocket(daemon)
|
|
2055
|
+
await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
2056
|
+
await daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})
|
|
2057
|
+
|
|
2058
|
+
expect(daemon.status().activeReleaseId).toBe("v3")
|
|
2059
|
+
expect(statusRelease(daemon, "v1").state).toBe("draining")
|
|
2060
|
+
} finally {
|
|
2061
|
+
if (socket) socket.close()
|
|
2062
|
+
await daemon.shutdown()
|
|
2063
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
2064
|
+
}
|
|
2065
|
+
})
|
|
2066
|
+
|
|
2067
|
+
test("status reports the drain classification of each draining generation", async () => {
|
|
2068
|
+
const fixture = await createFixture()
|
|
2069
|
+
const daemon = await startDaemon(fixture.config)
|
|
2070
|
+
/** @type {WebSocket | undefined} */
|
|
2071
|
+
let socket
|
|
2072
|
+
|
|
2073
|
+
try {
|
|
2074
|
+
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
2075
|
+
socket = await openWebSocket(daemon)
|
|
2076
|
+
await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
2077
|
+
|
|
2078
|
+
const draining = statusRelease(daemon, "v1")
|
|
2079
|
+
|
|
2080
|
+
expect(draining.state).toBe("draining")
|
|
2081
|
+
expect(draining.drain?.status).toBe("progressing")
|
|
2082
|
+
expect(draining.drain?.reason).toBe("1 live connection still owned by the generation")
|
|
2083
|
+
expect(statusRelease(daemon, "v2").drain).toBe(undefined)
|
|
2084
|
+
} finally {
|
|
2085
|
+
if (socket) socket.close()
|
|
2086
|
+
await daemon.shutdown()
|
|
2087
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
2088
|
+
}
|
|
2089
|
+
})
|
|
2090
|
+
|
|
2091
|
+
test("rollback refuses to restore a drained generation that still holds live processes", async () => {
|
|
2092
|
+
const fixture = await createFixture({nonBlockingDrainWorker: true, workerStopDelayMs: 6000})
|
|
2093
|
+
const config = normalizeConfig({
|
|
2094
|
+
...fixture.config,
|
|
2095
|
+
processes: fixture.config.processes.map((processConfig) => processConfig.id === "worker"
|
|
2096
|
+
? {...processConfig, gracefulStopMs: "indefinite"}
|
|
2097
|
+
: processConfig),
|
|
2098
|
+
proxy: {...fixture.config.proxy, nonBlockingDrainTimeoutMs: 200}
|
|
2099
|
+
})
|
|
2100
|
+
const daemon = await startDaemon(config)
|
|
2101
|
+
|
|
2102
|
+
try {
|
|
2103
|
+
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
2104
|
+
await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
2105
|
+
|
|
2106
|
+
// The retirement budget expires, Rollbridge records the generation as drained, and the
|
|
2107
|
+
// worker stays alive with its in-flight work — the shape a restore must not re-deploy.
|
|
2108
|
+
// The tracked process (the shell wrapper) is already gone, so the only honest record of
|
|
2109
|
+
// the leak is the process group that outlived it.
|
|
2110
|
+
await waitFor(() => statusRelease(daemon, "v1").state === "stopped")
|
|
2111
|
+
|
|
2112
|
+
const worker = statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")
|
|
2113
|
+
|
|
2114
|
+
if (!worker?.heldProcessGroup) {
|
|
2115
|
+
throw new Error(`the drained generation should still hold the worker's process group, got ${JSON.stringify(worker && {pid: worker.pid, retirementTimedOut: worker.retirementTimedOut, state: worker.state})}`)
|
|
2116
|
+
}
|
|
2117
|
+
|
|
2118
|
+
expect(worker.pid).toBeUndefined()
|
|
2119
|
+
expect(worker.retirementTimedOut).toBe(true)
|
|
2120
|
+
expect(worker.state).toBe("stopped")
|
|
2121
|
+
await expect(daemon.rollback({releaseId: "v1"})).rejects.toThrow(/recorded as drained while 1 process of that generation still hold live work \(worker: process group \d+ with \d+ live member\(s\) still alive after the tracked process exited\)\. Restoring it would re-register those processes under a new provenance and fail with a guardian provenance mismatch\. The guardian's active release is "v2"/)
|
|
2122
|
+
expect(daemon.status().activeReleaseId).toBe("v2")
|
|
2123
|
+
expect(await fetchText(daemon, "/release")).toBe("v2")
|
|
2124
|
+
} finally {
|
|
2125
|
+
await daemon.shutdown()
|
|
2126
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
2127
|
+
}
|
|
2128
|
+
})
|
|
2129
|
+
|
|
2130
|
+
test("deploy refuses to activate a release id whose draining generation still holds live processes", async () => {
|
|
2131
|
+
const fixture = await createFixture()
|
|
2132
|
+
const daemon = await startDaemon(fixture.config)
|
|
2133
|
+
/** @type {WebSocket | undefined} */
|
|
2134
|
+
let socket
|
|
2135
|
+
|
|
2136
|
+
try {
|
|
2137
|
+
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
2138
|
+
|
|
2139
|
+
// A held WebSocket keeps v1 draining after v2 takes traffic over, so v1's generation is
|
|
2140
|
+
// still alive when a restore names its id — the measured 2026-11-15 20:29Z and 20:41Z
|
|
2141
|
+
// shape, where the re-activation aborted with a bare guardian provenance mismatch.
|
|
2142
|
+
socket = await openWebSocket(daemon)
|
|
2143
|
+
await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
2144
|
+
|
|
2145
|
+
const draining = statusRelease(daemon, "v1")
|
|
2146
|
+
const heldPid = draining.processes.find((processStatus) => processStatus.id === "web")?.pid
|
|
2147
|
+
|
|
2148
|
+
if (!heldPid) throw new Error("the draining generation should still hold its web process")
|
|
2149
|
+
|
|
2150
|
+
await expect(daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})).rejects.toThrow(/Deploy refused before activation: release "v1" is retained in state draining while 1 process of that generation still hold live work \(web: state running, pid \d+\)\. Activating that release id again would re-register those processes under a new provenance and abort with a guardian provenance mismatch\. The guardian's active release is "v2"; deploy a new release id or reconcile the deployment pointer with `rollbridge status`\./)
|
|
2151
|
+
|
|
2152
|
+
const afterRefusal = daemon.status()
|
|
2153
|
+
|
|
2154
|
+
// Refusing before activation leaves one generation per release id: nothing was started,
|
|
2155
|
+
// the incumbent keeps serving, and the draining generation is still the one status reports.
|
|
2156
|
+
expect(afterRefusal.activeReleaseId).toBe("v2")
|
|
2157
|
+
expect(afterRefusal.releases.filter((release) => release.releaseId === "v1").length).toBe(1)
|
|
2158
|
+
expect(statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "web")?.pid).toBe(heldPid)
|
|
2159
|
+
expect(await fetchText(daemon, "/release")).toBe("v2")
|
|
2160
|
+
} finally {
|
|
2161
|
+
if (socket) socket.close()
|
|
2162
|
+
await daemon.shutdown()
|
|
2163
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
2164
|
+
}
|
|
2165
|
+
})
|
|
2166
|
+
|
|
2167
|
+
test("a restore deploy naming a drained generation that outlived its tracked process is refused by name", async () => {
|
|
2168
|
+
const fixture = await createFixture({nonBlockingDrainWorker: true, workerStopDelayMs: 6000})
|
|
2169
|
+
const config = normalizeConfig({
|
|
2170
|
+
...fixture.config,
|
|
2171
|
+
processes: fixture.config.processes.map((processConfig) => processConfig.id === "worker"
|
|
2172
|
+
? {...processConfig, gracefulStopMs: "indefinite"}
|
|
2173
|
+
: processConfig),
|
|
2174
|
+
proxy: {...fixture.config.proxy, nonBlockingDrainTimeoutMs: 200}
|
|
2175
|
+
})
|
|
2176
|
+
const daemon = await startDaemon(config)
|
|
2177
|
+
|
|
2178
|
+
try {
|
|
2179
|
+
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
2180
|
+
await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
2181
|
+
|
|
2182
|
+
// v1 is recorded as drained while its worker keeps its in-flight work in the group its
|
|
2183
|
+
// shell wrapper led. The 2026-10-25 carve-out that lets a dead generation's process key be
|
|
2184
|
+
// re-registered does not apply here: the group is alive, so the key is still held.
|
|
2185
|
+
await waitFor(() => statusRelease(daemon, "v1").state === "stopped")
|
|
2186
|
+
|
|
2187
|
+
const worker = statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")
|
|
2188
|
+
|
|
2189
|
+
if (!worker?.heldProcessGroup) {
|
|
2190
|
+
throw new Error(`the drained generation should still hold the worker's process group, got ${JSON.stringify(worker && {pid: worker.pid, retirementTimedOut: worker.retirementTimedOut, state: worker.state})}`)
|
|
2191
|
+
}
|
|
2192
|
+
|
|
2193
|
+
await expect(daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})).rejects.toThrow(/Deploy refused before activation: release "v1" is retained in state stopped while 1 process of that generation still hold live work \(worker: process group \d+ with \d+ live member\(s\) still alive after the tracked process exited\)\. Activating that release id again would re-register those processes under a new provenance and abort with a guardian provenance mismatch\. The guardian's active release is "v2"/)
|
|
2194
|
+
expect(daemon.status().activeReleaseId).toBe("v2")
|
|
2195
|
+
expect(daemon.status().releases.filter((release) => release.releaseId === "v1").length).toBe(1)
|
|
2196
|
+
expect(await fetchText(daemon, "/release")).toBe("v2")
|
|
2197
|
+
} finally {
|
|
2198
|
+
await daemon.shutdown()
|
|
2199
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
2200
|
+
}
|
|
2201
|
+
})
|
|
2202
|
+
|
|
2010
2203
|
test("the rollback control command switches traffic over the socket", async () => {
|
|
2011
2204
|
const fixture = await createFixture()
|
|
2012
2205
|
const daemon = await startDaemon(fixture.config)
|