rollbridge 0.1.63 → 0.1.66

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,199 @@
1
+ // @ts-check
2
+
3
+ import {describe, expect, test} from "@velocious/testing"
4
+ import {admitGenerationRelease, classifyDrainProgress} from "../src/drain-policy.js"
5
+
6
+ describe("drain-policy", () => {
7
+
8
+ const WINDOW_MS = 20000
9
+
10
+ /**
11
+ * @param {object} [options] - Observation shape.
12
+ * @param {number} [options.atMs] - Observation time.
13
+ * @param {number | undefined} [options.cpuMs] - Cumulative CPU of the group.
14
+ * @param {number | undefined} [options.heldProcessGroupPgid] - Group still alive after its tracked process exited.
15
+ * @param {number} [options.liveChildCount] - Runner children other than the group leader.
16
+ * @param {number | undefined} [options.pid] - Live pid, or undefined when the process is gone.
17
+ * @param {boolean} [options.retirementTimedOut] - Whether the supervisor already gave up on it.
18
+ * @returns {import("../src/drain-policy.js").DrainLivenessSample} One liveness observation.
19
+ */
20
+ function sample(options = {}) {
21
+ return {
22
+ atMs: options.atMs ?? 1000000,
23
+ processes: [{
24
+ cpuMs: "cpuMs" in options ? options.cpuMs : 5000,
25
+ heldProcessGroupPgid: options.heldProcessGroupPgid,
26
+ id: "worker",
27
+ liveChildCount: options.liveChildCount ?? 1,
28
+ pid: "pid" in options ? options.pid : 4242,
29
+ retirementTimedOut: options.retirementTimedOut ?? true
30
+ }]
31
+ }
32
+ }
33
+
34
+ /**
35
+ * @param {object} [options] - Classification inputs.
36
+ * @param {number} [options.connectionCount] - Live connections owned by the generation.
37
+ * @param {import("../src/drain-policy.js").DrainLivenessSample} [options.current] - Newest observation.
38
+ * @param {import("../src/drain-policy.js").DrainLivenessSample | undefined} [options.previous] - Older observation.
39
+ * @returns {import("../src/drain-policy.js").DrainClassification} Classification.
40
+ */
41
+ function classify(options = {}) {
42
+ return classifyDrainProgress({
43
+ connectionCount: options.connectionCount ?? 0,
44
+ currentSample: options.current ?? sample(),
45
+ minSampleWindowMs: WINDOW_MS,
46
+ previousSample: options.previous
47
+ })
48
+ }
49
+
50
+ test("a generation that still owns connections is draining, never wedged", () => {
51
+ const result = classify({connectionCount: 3, current: sample({liveChildCount: 0})})
52
+
53
+ expect(result.status).toBe("progressing")
54
+ expect(result.reason).toBe("3 live connections still owned by the generation")
55
+ })
56
+
57
+ test("a process still inside its retirement budget is draining", () => {
58
+ const result = classify({current: sample({liveChildCount: 0, retirementTimedOut: false})})
59
+
60
+ expect(result.status).toBe("progressing")
61
+ expect(result.reason).toBe("process worker is still inside its retirement budget")
62
+ })
63
+
64
+ test("a live process past its retirement budget with no runner child left is wedged", () => {
65
+ const result = classify({current: sample({liveChildCount: 0})})
66
+
67
+ expect(result.status).toBe("wedged")
68
+ expect(result.reason).toBe("1 live process past its retirement budget with no runner child remaining")
69
+ })
70
+
71
+ test("flat CPU across a sufficient window is wedged, and the window is reported", () => {
72
+ const previous = sample({atMs: 1000000, cpuMs: 5000})
73
+ const current = sample({atMs: 1025000, cpuMs: 5000, liveChildCount: 3})
74
+ const result = classify({current, previous})
75
+
76
+ expect(result.status).toBe("wedged")
77
+ expect(result.sampleWindowMs).toBe(25000)
78
+ expect(result.cpuDeltaMs).toBe(0)
79
+ expect(result.reason).toBe("1 live process with 3 runner children, no live connection, every retirement budget expired, CPU flat across 25000ms")
80
+ })
81
+
82
+ test("CPU that advanced across the window is a progressing drain", () => {
83
+ const previous = sample({atMs: 1000000, cpuMs: 5000})
84
+ const current = sample({atMs: 1025000, cpuMs: 9000})
85
+ const result = classify({current, previous})
86
+
87
+ expect(result.status).toBe("progressing")
88
+ expect(result.cpuDeltaMs).toBe(4000)
89
+ expect(result.reason).toBe("CPU advanced 4000ms across 25000ms")
90
+ })
91
+
92
+ test("CPU leaving the group with an exited member is a progressing drain", () => {
93
+ const previous = sample({atMs: 1000000, cpuMs: 9000})
94
+ const current = sample({atMs: 1025000, cpuMs: 5000})
95
+ const result = classify({current, previous})
96
+
97
+ expect(result.status).toBe("progressing")
98
+ expect(result.cpuDeltaMs).toBe(-4000)
99
+ expect(result.reason).toBe("the group changed (1 live process, -4000ms of consumed CPU left the group) across 25000ms")
100
+ })
101
+
102
+ test("the first observation cannot claim flatness", () => {
103
+ const result = classify({current: sample()})
104
+
105
+ expect(result.status).toBe("unproven")
106
+ expect(result.sampleWindowMs).toBe(0)
107
+ expect(result.reason).toBe("first liveness observation; a 20000ms window is required before CPU flatness can be claimed")
108
+ })
109
+
110
+ test("a window shorter than the minimum cannot claim flatness", () => {
111
+ const previous = sample({atMs: 1000000})
112
+ const current = sample({atMs: 1010000})
113
+ const result = classify({current, previous})
114
+
115
+ expect(result.status).toBe("unproven")
116
+ expect(result.sampleWindowMs).toBe(10000)
117
+ expect(result.reason).toBe("liveness window 10000ms is below the 20000ms minimum")
118
+ })
119
+
120
+ test("an unmeasurable CPU counter is unproven, never wedged", () => {
121
+ const previous = sample({atMs: 1000000, cpuMs: undefined})
122
+ const current = sample({atMs: 1025000, cpuMs: undefined})
123
+ const result = classify({current, previous})
124
+
125
+ expect(result.status).toBe("unproven")
126
+ expect(result.cpuDeltaMs).toBe(undefined)
127
+ expect(result.reason).toBe("process CPU is not measurable on this host")
128
+ })
129
+
130
+ test("a generation with no live process left is not a wedge", () => {
131
+ const result = classify({current: sample({pid: undefined})})
132
+
133
+ expect(result.status).toBe("progressing")
134
+ expect(result.reason).toBe("no live process remains in the generation")
135
+ })
136
+
137
+ test("a process group that outlived its tracked process is still a live process of the generation", () => {
138
+ const result = classify({current: sample({heldProcessGroupPgid: 4242, liveChildCount: 0, pid: undefined})})
139
+
140
+ expect(result.status).toBe("wedged")
141
+ expect(result.reason).toBe("1 live process past its retirement budget with no runner child remaining; 1 tracked process already exited and left worker pgid 4242 alive")
142
+ })
143
+
144
+ test("a held process group that still consumes CPU across the window is a progressing drain", () => {
145
+ const previous = sample({atMs: 1000000, cpuMs: 5000, heldProcessGroupPgid: 4242, liveChildCount: 2, pid: undefined})
146
+ const current = sample({atMs: 1025000, cpuMs: 9000, heldProcessGroupPgid: 4242, liveChildCount: 2, pid: undefined})
147
+ const result = classify({current, previous})
148
+
149
+ expect(result.status).toBe("progressing")
150
+ expect(result.cpuDeltaMs).toBe(4000)
151
+ expect(result.reason).toBe("CPU advanced 4000ms across 25000ms")
152
+ })
153
+
154
+ test("admission allows a handoff that stays within the bound and counts the incumbent", () => {
155
+ const admission = admitGenerationRelease({
156
+ candidateReleaseId: "v3",
157
+ draining: [{reason: "1 live connection still owned by the generation", releaseId: "v1", status: "progressing"}],
158
+ incumbentWillDrain: true,
159
+ maxConcurrentDrainingReleaseGroups: 2
160
+ })
161
+
162
+ expect(admission.allowed).toBe(true)
163
+ expect(admission.wouldBeDrainingCount).toBe(2)
164
+ expect(admission.reason).toBe(undefined)
165
+ })
166
+
167
+ test("admission refuses before activation and names every draining group with its classification", () => {
168
+ const admission = admitGenerationRelease({
169
+ candidateReleaseId: "20261115073717",
170
+ draining: [
171
+ {reason: "1 live process with 3 runner children, no live connection, every retirement budget expired, CPU flat across 25000ms", releaseId: "20261115063805", status: "wedged"},
172
+ {reason: "2 live connections still owned by the generation", releaseId: "20261114232900", status: "progressing"}
173
+ ],
174
+ incumbentWillDrain: true,
175
+ maxConcurrentDrainingReleaseGroups: 2
176
+ })
177
+
178
+ expect(admission.allowed).toBe(false)
179
+ expect(admission.wouldBeDrainingCount).toBe(3)
180
+ expect(admission.reason).toContain("Deploy refused before activation: 2 release groups already draining")
181
+ expect(admission.reason).toContain("20261115063805 wedged")
182
+ expect(admission.reason).toContain("Activating 20261115073717 would make 3, above the releaseRetention.maxConcurrentDrainingReleaseGroups bound of 2")
183
+ })
184
+
185
+ test("a first deploy with no incumbent adds no draining group", () => {
186
+ const admission = admitGenerationRelease({candidateReleaseId: "v1", draining: [], incumbentWillDrain: false, maxConcurrentDrainingReleaseGroups: 1})
187
+
188
+ expect(admission.allowed).toBe(true)
189
+ expect(admission.wouldBeDrainingCount).toBe(0)
190
+ })
191
+
192
+ test("a bound of 0 disables the admission check", () => {
193
+ const draining = Array.from({length: 30}, (_, index) => ({reason: "no runner child remaining", releaseId: `v${index}`, status: /** @type {const} */ ("wedged")}))
194
+ const admission = admitGenerationRelease({candidateReleaseId: "v31", draining, incumbentWillDrain: true, maxConcurrentDrainingReleaseGroups: 0})
195
+
196
+ expect(admission.allowed).toBe(true)
197
+ expect(admission.wouldBeDrainingCount).toBe(31)
198
+ })
199
+ })
@@ -5,7 +5,7 @@ import os from "node:os"
5
5
  import path from "node:path"
6
6
  import {describe, expect, test} from "@velocious/testing"
7
7
  import {fileURLToPath} from "node:url"
8
- import ManagedProcess from "../src/managed-process.js"
8
+ import ManagedProcess, {childRecordIsGone} from "../src/managed-process.js"
9
9
 
10
10
  describe("managed-process", () => {
11
11
 
@@ -930,4 +930,14 @@ test("the unlimited constant-delay fast path still applies maxDelayMs", () => {
930
930
 
931
931
  expect(queued).toBe(5)
932
932
  })
933
+
934
+ test("the guardian fence releases only a record that is terminal and holds no process group", () => {
935
+ expect(childRecordIsGone({heldProcessGroup: undefined, pid: undefined, state: "stopped"})).toBe(true)
936
+ expect(childRecordIsGone({heldProcessGroup: undefined, pid: undefined, state: "failed"})).toBe(true)
937
+ expect(childRecordIsGone({heldProcessGroup: undefined, pid: 4242, state: "running"})).toBe(false)
938
+
939
+ // The shape a drained generation leaves behind: the tracked process record is terminal, but the
940
+ // group it led still holds live members, so the key is still held by that generation.
941
+ expect(childRecordIsGone({heldProcessGroup: {liveMemberCount: 1, pgid: 4242}, pid: undefined, state: "stopped"})).toBe(false)
942
+ })
933
943
  })
@@ -4,7 +4,7 @@ import fs from "node:fs"
4
4
  import {describe, expect, test} from "@velocious/testing"
5
5
  import os from "node:os"
6
6
  import path from "node:path"
7
- import {measureProcessGroupRssBytes, processGroupHasLiveMembers, processGroupMembers} from "../src/process-memory.js"
7
+ import {measureProcessGroupRssBytes, processGroupHasLiveMembers, processGroupMembers, sampleProcessGroup} from "../src/process-memory.js"
8
8
 
9
9
  describe("process-memory", () => {
10
10
 
@@ -59,4 +59,50 @@ test("treats a process group containing only defunct members as stopped", () =>
59
59
  fs.rmSync(procPath, {force: true, recursive: true})
60
60
  }
61
61
  })
62
+
63
+ test("samples a process group's cumulative CPU and its live member count", () => {
64
+ const procPath = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-proc-cpu-"))
65
+
66
+ try {
67
+ // utime and stime are stat fields 14 and 15: index 11 and 12 after the parenthesised comm,
68
+ // so exactly eight placeholder fields sit between pgrp and utime.
69
+ fs.mkdirSync(path.join(procPath, "101"))
70
+ fs.writeFileSync(path.join(procPath, "101", "stat"), "101 (worker) S 1 77 0 0 0 0 0 0 0 0 300 200")
71
+ fs.mkdirSync(path.join(procPath, "102"))
72
+ fs.writeFileSync(path.join(procPath, "102", "stat"), "102 (runner) Z 1 77 0 0 0 0 0 0 0 0 10 5")
73
+ fs.mkdirSync(path.join(procPath, "103"))
74
+ fs.writeFileSync(path.join(procPath, "103", "stat"), "103 (other) S 1 99 0 0 0 0 0 0 0 0 9000 9000")
75
+
76
+ const sampled = sampleProcessGroup(77, procPath)
77
+
78
+ // Defunct members keep their consumed CPU, so the total covers both group members and
79
+ // excludes the foreign group 99 entirely: (300 + 200 + 10 + 5) ticks * 10ms per tick.
80
+ expect(sampled).toEqual({cpuMs: 5150, liveMemberCount: 1, memberCount: 2})
81
+ } finally {
82
+ fs.rmSync(procPath, {force: true, recursive: true})
83
+ }
84
+ })
85
+
86
+ test("reports group counts when a member's CPU fields are unreadable", () => {
87
+ const procPath = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-proc-cpu-partial-"))
88
+
89
+ try {
90
+ fs.mkdirSync(path.join(procPath, "101"))
91
+ fs.writeFileSync(path.join(procPath, "101", "stat"), "101 (worker) S 1 77 0 0")
92
+
93
+ expect(sampleProcessGroup(77, procPath)).toEqual({cpuMs: undefined, liveMemberCount: 1, memberCount: 1})
94
+ } finally {
95
+ fs.rmSync(procPath, {force: true, recursive: true})
96
+ }
97
+ })
98
+
99
+ test("reports a zero sample for a process group with no members", () => {
100
+ expect(sampleProcessGroup(2147483646)).toEqual({cpuMs: 0, liveMemberCount: 0, memberCount: 0})
101
+ })
102
+
103
+ linuxTest("samples the current process group's CPU as a positive measurable total", () => {
104
+ const sampled = sampleProcessGroup(currentProcessGroupId())
105
+
106
+ expect({value: Boolean(sampled && typeof sampled.cpuMs === "number" && sampled.cpuMs >= 0 && sampled.liveMemberCount >= 1), context: `expected a sampled group, got ${JSON.stringify(sampled)}`}).toMatchObject({value: true})
107
+ })
62
108
  })
@@ -1,6 +1,9 @@
1
1
  // @ts-check
2
2
 
3
3
  import {describe, expect, test} from "@velocious/testing"
4
+ import fs from "node:fs"
5
+ import os from "node:os"
6
+ import path from "node:path"
4
7
  import ReleaseGroup from "../src/release-group.js"
5
8
  import {normalizeConfig} from "../src/config.js"
6
9
 
@@ -9,9 +12,10 @@ describe("release-group", () => {
9
12
  /**
10
13
  * @param {import("../src/json.js").JsonValue} webProcess - The single proxied process definition.
11
14
  * @param {() => boolean} [shouldStart] - Whether process starts remain allowed.
15
+ * @param {string} [procPath] - Procfs root for drain liveness observations.
12
16
  * @returns {ReleaseGroup} A release group ready for buildProcess.
13
17
  */
14
- function buildRelease(webProcess, shouldStart = () => true) {
18
+ function buildRelease(webProcess, shouldStart = () => true, procPath = "/proc") {
15
19
  const config = normalizeConfig({
16
20
  application: "demo",
17
21
  control: {path: "/tmp/rollbridge-release-group.sock"},
@@ -19,7 +23,37 @@ function buildRelease(webProcess, shouldStart = () => true) {
19
23
  proxy: {host: "127.0.0.1", port: 0}
20
24
  })
21
25
 
22
- return new ReleaseGroup({config, logger: () => {}, releaseId: "v1", releasePath: "/tmp/rel", revision: "v1", shouldStart})
26
+ return new ReleaseGroup({config, logger: () => {}, procPath, releaseId: "v1", releasePath: "/tmp/rel", revision: "v1", shouldStart})
27
+ }
28
+
29
+ /**
30
+ * @param {string} procPath - Procfs root.
31
+ * @param {number} pid - Process id.
32
+ * @param {number} pgrp - Process group id.
33
+ * @param {string} state - Process state letter.
34
+ * @param {number} utimeTicks - Cumulative user CPU in clock ticks.
35
+ * @param {number} stimeTicks - Cumulative system CPU in clock ticks.
36
+ * @returns {void} One synthetic procfs member.
37
+ */
38
+ function writeProcMember(procPath, pid, pgrp, state, utimeTicks, stimeTicks) {
39
+ fs.mkdirSync(path.join(procPath, String(pid)), {recursive: true})
40
+
41
+ // Eight placeholder fields sit between pgrp and utime, which is stat field 14 (index 11
42
+ // after the parenthesised comm); stime is field 15 (index 12).
43
+ fs.writeFileSync(path.join(procPath, String(pid), "stat"), `${pid} (proc) ${state} 1 ${pgrp} 0 0 0 0 0 0 0 0 ${utimeTicks} ${stimeTicks}`)
44
+ }
45
+
46
+ /**
47
+ * @param {number | undefined} pid - Live pid, or undefined when the tracked process is gone.
48
+ * @param {import("../src/managed-process.js").HeldProcessGroup | undefined} [heldProcessGroup] - Group still alive after the tracked process exited.
49
+ * @returns {import("../src/managed-process.js").default} A process whose record is already terminal.
50
+ */
51
+ function drainedProcess(pid, heldProcessGroup = undefined) {
52
+ // eslint-disable-next-line local/no-unknown-jsdoc-type
53
+ return /** @type {import("../src/managed-process.js").default} */ (/** @type {unknown} */ ({
54
+ retirementTimedOut: true,
55
+ status: () => ({heldProcessGroup, pid, retirementTimedOut: true, state: "stopped"})
56
+ }))
23
57
  }
24
58
 
25
59
  test("templates interpolate values from the daemon environment", () => {
@@ -146,6 +180,7 @@ test("restoring a stopped release does not re-adopt a stale connection count", a
146
180
  cwd: "/tmp/rel",
147
181
  exitCode: 143,
148
182
  exitSignal: "SIGTERM",
183
+ heldProcessGroup: undefined,
149
184
  id: "web",
150
185
  lastMemoryRestartAt: undefined,
151
186
  lastStartReason: "deploy",
@@ -153,6 +188,7 @@ test("restoring a stopped release does not re-adopt a stale connection count", a
153
188
  memoryRestarts: 0,
154
189
  pid: undefined,
155
190
  restarts: 0,
191
+ retirementTimedOut: false,
156
192
  rssBytes: undefined,
157
193
  startedAt: "2026-10-08T06:22:23.500Z",
158
194
  state: "stopped",
@@ -163,6 +199,7 @@ test("restoring a stopped release does not re-adopt a stale connection count", a
163
199
  activatedAt: "2026-10-08T06:22:23.583Z",
164
200
  connectionCount: 14,
165
201
  connections: {http: 14, websocket: 0},
202
+ drain: undefined,
166
203
  drainStartedAt: "2026-10-08T07:58:00.464Z",
167
204
  ports: {web: 4200},
168
205
  processes: [processStatus],
@@ -178,4 +215,92 @@ test("restoring a stopped release does not re-adopt a stale connection count", a
178
215
  expect(release.status().connectionCount).toBe(0)
179
216
  expect(release.localConnections()).toEqual({http: 0, websocket: 0})
180
217
  })
218
+
219
+ test("status reports the drain classification of a draining generation and none of an active one", () => {
220
+ const procPath = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-drain-status-"))
221
+
222
+ try {
223
+ writeProcMember(procPath, 101, 101, "S", 300, 200)
224
+ writeProcMember(procPath, 102, 101, "S", 10, 5)
225
+
226
+ const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}}, () => true, procPath)
227
+
228
+ release.state = "draining"
229
+ release.processes.set("web", drainedProcess(101))
230
+
231
+ const first = release.status().drain
232
+
233
+ expect(first?.status).toBe("unproven")
234
+ expect(first?.sampleWindowMs).toBe(0)
235
+ expect(first?.reason).toBe("first liveness observation; a 20000ms window is required before CPU flatness can be claimed")
236
+
237
+ release.state = "active"
238
+
239
+ expect(release.status().drain).toBe(undefined)
240
+ } finally {
241
+ fs.rmSync(procPath, {force: true, recursive: true})
242
+ }
243
+ })
244
+
245
+ test("a drain observation is reused until a full sample window has passed", () => {
246
+ const procPath = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-drain-window-"))
247
+
248
+ try {
249
+ writeProcMember(procPath, 101, 101, "S", 300, 200)
250
+ writeProcMember(procPath, 102, 101, "S", 10, 5)
251
+
252
+ const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}}, () => true, procPath)
253
+
254
+ release.state = "draining"
255
+ release.processes.set("web", drainedProcess(101))
256
+
257
+ expect(release.status().drain?.status).toBe("unproven")
258
+
259
+ // Backdate the newest observation so the next read is a full window behind it.
260
+ release.drainNewestSample = {
261
+ atMs: Date.now() - 25000,
262
+ processes: [{cpuMs: 5150, heldProcessGroupPgid: undefined, id: "web", liveChildCount: 1, pid: 101, retirementTimedOut: true}]
263
+ }
264
+
265
+ const second = release.status().drain
266
+
267
+ expect(second?.status).toBe("wedged")
268
+ expect(second?.sampleWindowMs).toBe(25000)
269
+ expect(second?.cpuDeltaMs).toBe(0)
270
+
271
+ // The observation taken above is now the newest one. Changing the fixture inside the window
272
+ // must not produce a second observation: the classification still reports the reused sample,
273
+ // which is what keeps a procfs scan off the status path.
274
+ writeProcMember(procPath, 101, 101, "S", 900, 600)
275
+
276
+ const third = release.status().drain
277
+
278
+ expect(third?.status).toBe("wedged")
279
+ expect(third?.sampleWindowMs).toBe(25000)
280
+ expect(third?.cpuDeltaMs).toBe(0)
281
+ } finally {
282
+ fs.rmSync(procPath, {force: true, recursive: true})
283
+ }
284
+ })
285
+
286
+ test("a generation still holding a process group after its tracked process exited is observed through that group", () => {
287
+ const procPath = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-drain-held-"))
288
+
289
+ try {
290
+ // The group leader exited, so only the member that outlived it is left in the group.
291
+ writeProcMember(procPath, 102, 101, "S", 300, 200)
292
+
293
+ const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}}, () => true, procPath)
294
+
295
+ release.state = "draining"
296
+ release.processes.set("web", drainedProcess(undefined, {liveMemberCount: 1, pgid: 101}))
297
+
298
+ const drain = release.status().drain
299
+
300
+ expect(drain?.status).toBe("wedged")
301
+ expect(drain?.reason).toBe("1 live process past its retirement budget with no runner child remaining; 1 tracked process already exited and left web pgid 101 alive")
302
+ } finally {
303
+ fs.rmSync(procPath, {force: true, recursive: true})
304
+ }
305
+ })
181
306
  })
@@ -2007,6 +2007,199 @@ test("rollback to a still-draining release stops the old instance instead of orp
2007
2007
  }
2008
2008
  })
2009
2009
 
2010
+ test("deploy is refused before activation once the draining-generation bound is reached", async () => {
2011
+ const fixture = await createFixture()
2012
+ const config = normalizeConfig({...fixture.config, releaseRetention: {keep: 10, maxAgeMs: 0, maxConcurrentDrainingReleaseGroups: 1}})
2013
+ const daemon = await startDaemon(config)
2014
+ /** @type {WebSocket | undefined} */
2015
+ let socket
2016
+
2017
+ try {
2018
+ await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
2019
+
2020
+ // A held WebSocket keeps v1 draining after v2 takes traffic over: the measured production
2021
+ // shape where a drain outlasts the deploy cadence.
2022
+ socket = await openWebSocket(daemon)
2023
+ await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
2024
+
2025
+ expect(statusRelease(daemon, "v1").state).toBe("draining")
2026
+
2027
+ const beforeRefusal = daemon.status()
2028
+
2029
+ await expect(daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})).rejects.toThrow(/already draining \(v1 progressing \(1 live connection still owned by the generation\)\)\. Activating v3 would make 2, above the releaseRetention\.maxConcurrentDrainingReleaseGroups bound of 1/)
2030
+
2031
+ const afterRefusal = daemon.status()
2032
+
2033
+ // Refusing pre-handoff means nothing was started: no new generation, incumbent intact.
2034
+ expect(afterRefusal.activeReleaseId).toBe("v2")
2035
+ expect(afterRefusal.releases.length).toBe(beforeRefusal.releases.length)
2036
+ expect(afterRefusal.releases.some((release) => release.releaseId === "v3")).toBe(false)
2037
+ expect(await fetchText(daemon, "/release")).toBe("v2")
2038
+ } finally {
2039
+ if (socket) socket.close()
2040
+ await daemon.shutdown()
2041
+ await fs.rm(fixture.root, {force: true, recursive: true})
2042
+ }
2043
+ })
2044
+
2045
+ test("a draining-generation bound of 0 leaves concurrent drains unbounded", async () => {
2046
+ const fixture = await createFixture()
2047
+ const config = normalizeConfig({...fixture.config, releaseRetention: {keep: 10, maxAgeMs: 0, maxConcurrentDrainingReleaseGroups: 0}})
2048
+ const daemon = await startDaemon(config)
2049
+ /** @type {WebSocket | undefined} */
2050
+ let socket
2051
+
2052
+ try {
2053
+ await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
2054
+ socket = await openWebSocket(daemon)
2055
+ await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
2056
+ await daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})
2057
+
2058
+ expect(daemon.status().activeReleaseId).toBe("v3")
2059
+ expect(statusRelease(daemon, "v1").state).toBe("draining")
2060
+ } finally {
2061
+ if (socket) socket.close()
2062
+ await daemon.shutdown()
2063
+ await fs.rm(fixture.root, {force: true, recursive: true})
2064
+ }
2065
+ })
2066
+
2067
+ test("status reports the drain classification of each draining generation", async () => {
2068
+ const fixture = await createFixture()
2069
+ const daemon = await startDaemon(fixture.config)
2070
+ /** @type {WebSocket | undefined} */
2071
+ let socket
2072
+
2073
+ try {
2074
+ await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
2075
+ socket = await openWebSocket(daemon)
2076
+ await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
2077
+
2078
+ const draining = statusRelease(daemon, "v1")
2079
+
2080
+ expect(draining.state).toBe("draining")
2081
+ expect(draining.drain?.status).toBe("progressing")
2082
+ expect(draining.drain?.reason).toBe("1 live connection still owned by the generation")
2083
+ expect(statusRelease(daemon, "v2").drain).toBe(undefined)
2084
+ } finally {
2085
+ if (socket) socket.close()
2086
+ await daemon.shutdown()
2087
+ await fs.rm(fixture.root, {force: true, recursive: true})
2088
+ }
2089
+ })
2090
+
2091
+ test("rollback refuses to restore a drained generation that still holds live processes", async () => {
2092
+ const fixture = await createFixture({nonBlockingDrainWorker: true, workerStopDelayMs: 6000})
2093
+ const config = normalizeConfig({
2094
+ ...fixture.config,
2095
+ processes: fixture.config.processes.map((processConfig) => processConfig.id === "worker"
2096
+ ? {...processConfig, gracefulStopMs: "indefinite"}
2097
+ : processConfig),
2098
+ proxy: {...fixture.config.proxy, nonBlockingDrainTimeoutMs: 200}
2099
+ })
2100
+ const daemon = await startDaemon(config)
2101
+
2102
+ try {
2103
+ await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
2104
+ await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
2105
+
2106
+ // The retirement budget expires, Rollbridge records the generation as drained, and the
2107
+ // worker stays alive with its in-flight work — the shape a restore must not re-deploy.
2108
+ // The tracked process (the shell wrapper) is already gone, so the only honest record of
2109
+ // the leak is the process group that outlived it.
2110
+ await waitFor(() => statusRelease(daemon, "v1").state === "stopped")
2111
+
2112
+ const worker = statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")
2113
+
2114
+ if (!worker?.heldProcessGroup) {
2115
+ throw new Error(`the drained generation should still hold the worker's process group, got ${JSON.stringify(worker && {pid: worker.pid, retirementTimedOut: worker.retirementTimedOut, state: worker.state})}`)
2116
+ }
2117
+
2118
+ expect(worker.pid).toBeUndefined()
2119
+ expect(worker.retirementTimedOut).toBe(true)
2120
+ expect(worker.state).toBe("stopped")
2121
+ await expect(daemon.rollback({releaseId: "v1"})).rejects.toThrow(/recorded as drained while 1 process of that generation still hold live work \(worker: process group \d+ with \d+ live member\(s\) still alive after the tracked process exited\)\. Restoring it would re-register those processes under a new provenance and fail with a guardian provenance mismatch\. The guardian's active release is "v2"/)
2122
+ expect(daemon.status().activeReleaseId).toBe("v2")
2123
+ expect(await fetchText(daemon, "/release")).toBe("v2")
2124
+ } finally {
2125
+ await daemon.shutdown()
2126
+ await fs.rm(fixture.root, {force: true, recursive: true})
2127
+ }
2128
+ })
2129
+
2130
+ test("deploy refuses to activate a release id whose draining generation still holds live processes", async () => {
2131
+ const fixture = await createFixture()
2132
+ const daemon = await startDaemon(fixture.config)
2133
+ /** @type {WebSocket | undefined} */
2134
+ let socket
2135
+
2136
+ try {
2137
+ await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
2138
+
2139
+ // A held WebSocket keeps v1 draining after v2 takes traffic over, so v1's generation is
2140
+ // still alive when a restore names its id — the measured 2026-11-15 20:29Z and 20:41Z
2141
+ // shape, where the re-activation aborted with a bare guardian provenance mismatch.
2142
+ socket = await openWebSocket(daemon)
2143
+ await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
2144
+
2145
+ const draining = statusRelease(daemon, "v1")
2146
+ const heldPid = draining.processes.find((processStatus) => processStatus.id === "web")?.pid
2147
+
2148
+ if (!heldPid) throw new Error("the draining generation should still hold its web process")
2149
+
2150
+ await expect(daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})).rejects.toThrow(/Deploy refused before activation: release "v1" is retained in state draining while 1 process of that generation still hold live work \(web: state running, pid \d+\)\. Activating that release id again would re-register those processes under a new provenance and abort with a guardian provenance mismatch\. The guardian's active release is "v2"; deploy a new release id or reconcile the deployment pointer with `rollbridge status`\./)
2151
+
2152
+ const afterRefusal = daemon.status()
2153
+
2154
+ // Refusing before activation leaves one generation per release id: nothing was started,
2155
+ // the incumbent keeps serving, and the draining generation is still the one status reports.
2156
+ expect(afterRefusal.activeReleaseId).toBe("v2")
2157
+ expect(afterRefusal.releases.filter((release) => release.releaseId === "v1").length).toBe(1)
2158
+ expect(statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "web")?.pid).toBe(heldPid)
2159
+ expect(await fetchText(daemon, "/release")).toBe("v2")
2160
+ } finally {
2161
+ if (socket) socket.close()
2162
+ await daemon.shutdown()
2163
+ await fs.rm(fixture.root, {force: true, recursive: true})
2164
+ }
2165
+ })
2166
+
2167
+ test("a restore deploy naming a drained generation that outlived its tracked process is refused by name", async () => {
2168
+ const fixture = await createFixture({nonBlockingDrainWorker: true, workerStopDelayMs: 6000})
2169
+ const config = normalizeConfig({
2170
+ ...fixture.config,
2171
+ processes: fixture.config.processes.map((processConfig) => processConfig.id === "worker"
2172
+ ? {...processConfig, gracefulStopMs: "indefinite"}
2173
+ : processConfig),
2174
+ proxy: {...fixture.config.proxy, nonBlockingDrainTimeoutMs: 200}
2175
+ })
2176
+ const daemon = await startDaemon(config)
2177
+
2178
+ try {
2179
+ await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
2180
+ await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
2181
+
2182
+ // v1 is recorded as drained while its worker keeps its in-flight work in the group its
2183
+ // shell wrapper led. The 2026-10-25 carve-out that lets a dead generation's process key be
2184
+ // re-registered does not apply here: the group is alive, so the key is still held.
2185
+ await waitFor(() => statusRelease(daemon, "v1").state === "stopped")
2186
+
2187
+ const worker = statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")
2188
+
2189
+ if (!worker?.heldProcessGroup) {
2190
+ throw new Error(`the drained generation should still hold the worker's process group, got ${JSON.stringify(worker && {pid: worker.pid, retirementTimedOut: worker.retirementTimedOut, state: worker.state})}`)
2191
+ }
2192
+
2193
+ await expect(daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})).rejects.toThrow(/Deploy refused before activation: release "v1" is retained in state stopped while 1 process of that generation still hold live work \(worker: process group \d+ with \d+ live member\(s\) still alive after the tracked process exited\)\. Activating that release id again would re-register those processes under a new provenance and abort with a guardian provenance mismatch\. The guardian's active release is "v2"/)
2194
+ expect(daemon.status().activeReleaseId).toBe("v2")
2195
+ expect(daemon.status().releases.filter((release) => release.releaseId === "v1").length).toBe(1)
2196
+ expect(await fetchText(daemon, "/release")).toBe("v2")
2197
+ } finally {
2198
+ await daemon.shutdown()
2199
+ await fs.rm(fixture.root, {force: true, recursive: true})
2200
+ }
2201
+ })
2202
+
2010
2203
  test("the rollback control command switches traffic over the socket", async () => {
2011
2204
  const fixture = await createFixture()
2012
2205
  const daemon = await startDaemon(fixture.config)