rollbridge 0.1.70 → 0.1.71

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/README.md +10 -63
  2. package/TODO.md +0 -13
  3. package/docs/cli.md +3 -128
  4. package/docs/config.md +6 -177
  5. package/docs/troubleshooting.md +6 -97
  6. package/docs/velocious.md +1 -1
  7. package/docs/workers.md +0 -11
  8. package/examples/tensorbuzz.com.js +2 -3
  9. package/package.json +1 -1
  10. package/src/cli.js +2 -68
  11. package/src/config.js +8 -347
  12. package/src/daemon.js +65 -387
  13. package/src/doctor.js +0 -74
  14. package/src/guardian-client.js +8 -67
  15. package/src/health.js +8 -99
  16. package/src/managed-process.js +16 -197
  17. package/src/process-guardian.js +22 -212
  18. package/src/process-memory.js +2 -62
  19. package/src/release-group.js +37 -435
  20. package/tensorbuzz.yml +1 -20
  21. package/test/completion.test.js +2 -5
  22. package/test/config-validation.test.js +5 -135
  23. package/test/guardian-client.test.js +5 -119
  24. package/test/health.test.js +1 -74
  25. package/test/managed-process.test.js +1 -260
  26. package/test/owner-recovery.test.js +21 -125
  27. package/test/owner-replacement.test.js +14 -175
  28. package/test/process-memory.test.js +1 -47
  29. package/test/release-group.test.js +2 -408
  30. package/test/rollbridge.test.js +72 -737
  31. package/test/shutdown-completion.test.js +0 -52
  32. package/test/support/process.js +1 -3
  33. package/changelog.d/20261025-guardian-re-register-dead-generation.md +0 -13
  34. package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +0 -1
  35. package/changelog.d/20261104-compose-service-units.md +0 -21
  36. package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +0 -20
  37. package/changelog.d/20261106-release-dependency-boot-budgets.md +0 -32
  38. package/changelog.d/20261109-path-filtered-builds.md +0 -11
  39. package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +0 -26
  40. package/changelog.d/20261111-contention-sensitive-daemon-specs.md +0 -27
  41. package/changelog.d/20261113-bounded-nonblocking-retirement.md +0 -5
  42. package/changelog.d/20261115-bound-draining-release-groups.md +0 -62
  43. package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +0 -27
  44. package/changelog.d/20261115-wedged-generation-escalation.md +0 -44
  45. package/changelog.d/20261116-bounded-explicit-stop-retirement.md +0 -30
  46. package/changelog.d/20261116-config-json-canonical-shape.md +0 -11
  47. package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +0 -29
  48. package/changelog.d/20261116-guardian-version-handoff.md +0 -37
  49. package/changelog.d/20261116-reclaim-dead-owner-mutation.md +0 -7
  50. package/changelog.d/20261117-activate-before-incumbent-retirement.md +0 -1
  51. package/changelog.d/20261117-activation-path-drain-deadline.md +0 -35
  52. package/changelog.d/20261117-stop-companion-after-retired-dependency.md +0 -7
  53. package/docs/compose-services.md +0 -222
  54. package/src/compose-unit.js +0 -725
  55. package/src/drain-policy.js +0 -252
  56. package/src/process-order.js +0 -26
  57. package/test/compose-config.test.js +0 -160
  58. package/test/compose-docker.test.js +0 -297
  59. package/test/compose-unit.test.js +0 -415
  60. package/test/drain-policy.test.js +0 -328
  61. package/test/fixtures/deferred-boot-app.js +0 -106
  62. package/test/fixtures/gated-app.js +0 -50
  63. package/test/tensorbuzz-config.test.js +0 -228
@@ -1,9 +1,6 @@
1
1
  // @ts-check
2
2
 
3
3
  import {describe, expect, test} from "@velocious/testing"
4
- import fs from "node:fs"
5
- import os from "node:os"
6
- import path from "node:path"
7
4
  import ReleaseGroup from "../src/release-group.js"
8
5
  import {normalizeConfig} from "../src/config.js"
9
6
 
@@ -12,10 +9,9 @@ describe("release-group", () => {
12
9
  /**
13
10
  * @param {import("../src/json.js").JsonValue} webProcess - The single proxied process definition.
14
11
  * @param {() => boolean} [shouldStart] - Whether process starts remain allowed.
15
- * @param {string} [procPath] - Procfs root for drain liveness observations.
16
12
  * @returns {ReleaseGroup} A release group ready for buildProcess.
17
13
  */
18
- function buildRelease(webProcess, shouldStart = () => true, procPath = "/proc") {
14
+ function buildRelease(webProcess, shouldStart = () => true) {
19
15
  const config = normalizeConfig({
20
16
  application: "demo",
21
17
  control: {path: "/tmp/rollbridge-release-group.sock"},
@@ -23,37 +19,7 @@ function buildRelease(webProcess, shouldStart = () => true, procPath = "/proc")
23
19
  proxy: {host: "127.0.0.1", port: 0}
24
20
  })
25
21
 
26
- return new ReleaseGroup({config, logger: () => {}, procPath, releaseId: "v1", releasePath: "/tmp/rel", revision: "v1", shouldStart})
27
- }
28
-
29
- /**
30
- * @param {string} procPath - Procfs root.
31
- * @param {number} pid - Process id.
32
- * @param {number} pgrp - Process group id.
33
- * @param {string} state - Process state letter.
34
- * @param {number} utimeTicks - Cumulative user CPU in clock ticks.
35
- * @param {number} stimeTicks - Cumulative system CPU in clock ticks.
36
- * @returns {void} One synthetic procfs member.
37
- */
38
- function writeProcMember(procPath, pid, pgrp, state, utimeTicks, stimeTicks) {
39
- fs.mkdirSync(path.join(procPath, String(pid)), {recursive: true})
40
-
41
- // Eight placeholder fields sit between pgrp and utime, which is stat field 14 (index 11
42
- // after the parenthesised comm); stime is field 15 (index 12).
43
- fs.writeFileSync(path.join(procPath, String(pid), "stat"), `${pid} (proc) ${state} 1 ${pgrp} 0 0 0 0 0 0 0 0 ${utimeTicks} ${stimeTicks}`)
44
- }
45
-
46
- /**
47
- * @param {number | undefined} pid - Live pid, or undefined when the tracked process is gone.
48
- * @param {import("../src/managed-process.js").HeldProcessGroup | undefined} [heldProcessGroup] - Group still alive after the tracked process exited.
49
- * @returns {import("../src/managed-process.js").default} A process whose record is already terminal.
50
- */
51
- function drainedProcess(pid, heldProcessGroup = undefined) {
52
- // eslint-disable-next-line local/no-unknown-jsdoc-type
53
- return /** @type {import("../src/managed-process.js").default} */ (/** @type {unknown} */ ({
54
- retirementTimedOut: true,
55
- status: () => ({heldProcessGroup, pid, retirementTimedOut: true, state: "stopped"})
56
- }))
22
+ return new ReleaseGroup({config, logger: () => {}, releaseId: "v1", releasePath: "/tmp/rel", revision: "v1", shouldStart})
57
23
  }
58
24
 
59
25
  test("templates interpolate values from the daemon environment", () => {
@@ -99,42 +65,6 @@ test("replica processes get a replica index, count, and template context", () =>
99
65
  expect(replica.env.SLOT).toBe("1")
100
66
  })
101
67
 
102
- test("a companion stops without restarting after its same-generation service dependency retires", async () => {
103
- const config = normalizeConfig({
104
- application: "demo",
105
- control: {path: "/tmp/rollbridge-release-group.sock"},
106
- processes: [
107
- {
108
- command: "run jobs-main",
109
- id: "jobs-main",
110
- lifecycle: {quietCommand: "retire jobs-main"},
111
- policy: "service",
112
- port: {from: 7331, to: 7399}
113
- },
114
- {command: "run worker", dependsOn: ["jobs-main"], id: "worker", policy: "companion"},
115
- {command: "run web", id: "web", policy: "proxied", port: {from: 14500, to: 14599}}
116
- ],
117
- proxy: {host: "127.0.0.1", port: 0}
118
- })
119
- const release = new ReleaseGroup({config, logger: () => {}, releaseId: "v1", releasePath: "/tmp/rel", revision: "v1"})
120
- const jobsMain = release.buildProcess(config.processes[0])
121
- const worker = release.buildProcess(config.processes[1])
122
-
123
- release.state = "active"
124
- release.processes.set("jobs-main", jobsMain)
125
- release.processes.set("worker", worker)
126
-
127
- await jobsMain.setLifecycleRole("retired")
128
- worker.command = `${JSON.stringify(process.execPath)} -e ${JSON.stringify("process.exit(1)")}`
129
- worker.cwd = undefined
130
- await new Promise((resolve, reject) => {
131
- worker.once("exit", resolve)
132
- worker.start("deploy").catch(reject)
133
- })
134
-
135
- expect(worker.status()).toMatchObject({restarts: 0, state: "stopped"})
136
- })
137
-
138
68
  test("a referenced daemon environment variable that is unset fails fast", async () => {
139
69
  const release = buildRelease({
140
70
  command: "run {{env.ROLLBRIDGE_ENV_MISSING}}",
@@ -179,96 +109,6 @@ test("stopping a release abandons its connection count instead of persisting a s
179
109
  expect(release.localConnections()).toEqual({http: 0, websocket: 0})
180
110
  })
181
111
 
182
- test("a wedged non-blocking process cannot block release retirement", async () => {
183
- /** @type {Array<{message: string, data?: import("../src/json.js").JsonValue}>} */
184
- const logs = []
185
- const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}})
186
- release.logger = (message, data) => logs.push({message, data})
187
- release.state = "draining"
188
- release.nonBlockingDrainIds.add("worker")
189
- release.handoffServiceIds.add("jobs-main")
190
- // eslint-disable-next-line local/no-unknown-jsdoc-type
191
- release.processes.set("worker", /** @type {import("../src/managed-process.js").default} */ (/** @type {unknown} */ ({
192
- quiesceStrict: async () => {},
193
- status: () => ({pid: 4242}),
194
- retirementTimedOut: true,
195
- stop: async () => {}
196
- })))
197
- let handoffStopped = false
198
- // eslint-disable-next-line local/no-unknown-jsdoc-type
199
- release.processes.set("jobs-main", /** @type {import("../src/managed-process.js").default} */ (/** @type {unknown} */ ({
200
- stop: async () => { handoffStopped = true }
201
- })))
202
-
203
- await release.drainAndStop(0, {...release.config, proxy: {...release.config.proxy, nonBlockingDrainTimeoutMs: 20}})
204
-
205
- expect(release.state).toBe("stopped")
206
- expect(handoffStopped).toBe(true)
207
- expect(logs).toMatchObject([{message: "release non-blocking process still draining", data: {pid: 4242, processId: "worker"}}])
208
- })
209
-
210
- test("an explicit stop bounds the processes it must not kill and reports what it left alive", async () => {
211
- /** @type {Array<{message: string, data?: import("../src/json.js").JsonValue}>} */
212
- const logs = []
213
- /** @type {Array<{id: string, options: {retirementTimeoutMs?: number, timeoutMs?: number}}>} */
214
- const stopOptions = []
215
- const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}})
216
-
217
- release.logger = (message, data) => logs.push({message, data})
218
- release.nonBlockingDrainIds.add("worker")
219
-
220
- /**
221
- * @param {string} id - Process id.
222
- * @param {boolean} unboundedWindow - Whether its graceful window can ever end.
223
- * @returns {void} A process recorded in the generation.
224
- */
225
- const addProcess = (id, unboundedWindow) => {
226
- const instance = {
227
- hasUnboundedGracefulWindow: () => unboundedWindow,
228
- retirementTimedOut: false,
229
- status: () => ({pid: 4242, state: "stopped"}),
230
- stop: async (/** @type {{retirementTimeoutMs?: number, timeoutMs?: number}} */ options) => {
231
- stopOptions.push({id, options})
232
- // Only the retirement branch sets this, so a process that was never budgeted never claims it.
233
- if (options.retirementTimeoutMs !== undefined) instance.retirementTimedOut = true
234
- }
235
- }
236
-
237
- // eslint-disable-next-line local/no-unknown-jsdoc-type
238
- release.processes.set(id, /** @type {import("../src/managed-process.js").default} */ (/** @type {unknown} */ (instance)))
239
- }
240
-
241
- addProcess("worker", false)
242
- addProcess("jobs-main", true)
243
- addProcess("web", false)
244
-
245
- await release.stop({retirementTimeoutMs: 40})
246
-
247
- expect(stopOptions).toEqual([
248
- {id: "worker", options: {retirementTimeoutMs: 40}},
249
- {id: "jobs-main", options: {retirementTimeoutMs: 40}},
250
- {id: "web", options: {}}
251
- ])
252
- expect(release.state).toBe("stopped")
253
- expect(logs).toMatchObject([
254
- {message: "release stop gave up waiting for a retiring process", data: {pid: 4242, processId: "worker", releaseId: "v1", timeoutMs: 40}},
255
- {message: "release stop gave up waiting for a retiring process", data: {pid: 4242, processId: "jobs-main", releaseId: "v1", timeoutMs: 40}}
256
- ])
257
-
258
- // Shutdown has no budget to hand out and must still wait for the processes it owns, so an
259
- // unbudgeted stop changes nothing.
260
- logs.length = 0
261
- stopOptions.length = 0
262
- await release.stop()
263
-
264
- expect(stopOptions).toEqual([
265
- {id: "worker", options: {}},
266
- {id: "jobs-main", options: {}},
267
- {id: "web", options: {}}
268
- ])
269
- expect(logs).toHaveLength(0)
270
- })
271
-
272
112
  test("restoring a stopped release does not re-adopt a stale connection count", async () => {
273
113
  const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}})
274
114
 
@@ -278,7 +118,6 @@ test("restoring a stopped release does not re-adopt a stale connection count", a
278
118
  cwd: "/tmp/rel",
279
119
  exitCode: 143,
280
120
  exitSignal: "SIGTERM",
281
- heldProcessGroup: undefined,
282
121
  id: "web",
283
122
  lastMemoryRestartAt: undefined,
284
123
  lastStartReason: "deploy",
@@ -286,7 +125,6 @@ test("restoring a stopped release does not re-adopt a stale connection count", a
286
125
  memoryRestarts: 0,
287
126
  pid: undefined,
288
127
  restarts: 0,
289
- retirementTimedOut: false,
290
128
  rssBytes: undefined,
291
129
  startedAt: "2026-10-08T06:22:23.500Z",
292
130
  state: "stopped",
@@ -297,7 +135,6 @@ test("restoring a stopped release does not re-adopt a stale connection count", a
297
135
  activatedAt: "2026-10-08T06:22:23.583Z",
298
136
  connectionCount: 14,
299
137
  connections: {http: 14, websocket: 0},
300
- drain: undefined,
301
138
  drainStartedAt: "2026-10-08T07:58:00.464Z",
302
139
  ports: {web: 4200},
303
140
  processes: [processStatus],
@@ -313,247 +150,4 @@ test("restoring a stopped release does not re-adopt a stale connection count", a
313
150
  expect(release.status().connectionCount).toBe(0)
314
151
  expect(release.localConnections()).toEqual({http: 0, websocket: 0})
315
152
  })
316
-
317
- test("status reports the drain classification of a draining generation and none of an active one", () => {
318
- const procPath = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-drain-status-"))
319
-
320
- try {
321
- writeProcMember(procPath, 101, 101, "S", 300, 200)
322
- writeProcMember(procPath, 102, 101, "S", 10, 5)
323
-
324
- const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}}, () => true, procPath)
325
-
326
- release.state = "draining"
327
- release.processes.set("web", drainedProcess(101))
328
-
329
- const first = release.status().drain
330
-
331
- expect(first?.status).toBe("unproven")
332
- expect(first?.sampleWindowMs).toBe(0)
333
- expect(first?.reason).toBe("first liveness observation; a 20000ms window is required before CPU flatness can be claimed")
334
-
335
- release.state = "active"
336
-
337
- expect(release.status().drain).toBe(undefined)
338
- } finally {
339
- fs.rmSync(procPath, {force: true, recursive: true})
340
- }
341
- })
342
-
343
- test("a draining generation that still owns a connection past its drain budget is named timedOut, not left claiming progress", () => {
344
- const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}}, () => true)
345
-
346
- release.state = "draining"
347
- release.connectionCount = 1
348
- // The connection wait was given up at proxy.drainTimeoutMs (60000ms default) 30 seconds ago.
349
- release.drainStartedAt = new Date(Date.now() - 90000).toISOString()
350
-
351
- const pastBudget = release.status().drain
352
-
353
- expect(pastBudget?.status).toBe("timedOut")
354
- expect(pastBudget?.reason).toMatch(/^drain budget 60000ms exceeded by \d+ms with 1 live connection still owned by the generation$/)
355
-
356
- release.drainStartedAt = new Date().toISOString()
357
-
358
- const insideBudget = release.status().drain
359
-
360
- expect(insideBudget?.status).toBe("progressing")
361
- expect(insideBudget?.reason).toBe("1 live connection still owned by the generation")
362
- })
363
-
364
- test("a drain observation is reused until a full sample window has passed", () => {
365
- const procPath = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-drain-window-"))
366
-
367
- try {
368
- writeProcMember(procPath, 101, 101, "S", 300, 200)
369
- writeProcMember(procPath, 102, 101, "S", 10, 5)
370
-
371
- const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}}, () => true, procPath)
372
-
373
- release.state = "draining"
374
- release.processes.set("web", drainedProcess(101))
375
-
376
- expect(release.status().drain?.status).toBe("unproven")
377
-
378
- // Backdate the newest observation so the next read is a full window behind it.
379
- release.drainNewestSample = {
380
- atMs: Date.now() - 25000,
381
- processes: [{cpuMs: 5150, heldProcessGroupPgid: undefined, id: "web", liveChildCount: 1, pid: 101, retirementTimedOut: true}]
382
- }
383
-
384
- const second = release.status().drain
385
-
386
- expect(second?.status).toBe("wedged")
387
- expect(second?.sampleWindowMs).toBe(25000)
388
- expect(second?.cpuDeltaMs).toBe(0)
389
-
390
- // The observation taken above is now the newest one. Changing the fixture inside the window
391
- // must not produce a second observation: the classification still reports the reused sample,
392
- // which is what keeps a procfs scan off the status path.
393
- writeProcMember(procPath, 101, 101, "S", 900, 600)
394
-
395
- const third = release.status().drain
396
-
397
- expect(third?.status).toBe("wedged")
398
- expect(third?.sampleWindowMs).toBe(25000)
399
- expect(third?.cpuDeltaMs).toBe(0)
400
- } finally {
401
- fs.rmSync(procPath, {force: true, recursive: true})
402
- }
403
- })
404
-
405
- test("a generation still holding a process group after its tracked process exited is observed through that group", () => {
406
- const procPath = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-drain-held-"))
407
-
408
- try {
409
- // The group leader exited, so only the member that outlived it is left in the group.
410
- writeProcMember(procPath, 102, 101, "S", 300, 200)
411
-
412
- const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}}, () => true, procPath)
413
-
414
- release.state = "draining"
415
- release.processes.set("web", drainedProcess(undefined, {liveMemberCount: 1, pgid: 101}))
416
-
417
- const drain = release.status().drain
418
-
419
- expect(drain?.status).toBe("wedged")
420
- expect(drain?.reason).toBe("1 live process past its retirement budget with no runner child remaining; 1 tracked process already exited and left web pgid 101 alive")
421
- } finally {
422
- fs.rmSync(procPath, {force: true, recursive: true})
423
- }
424
- })
425
-
426
- /**
427
- * @param {number} pid - Live pid the supervisor can still signal.
428
- * @param {string[]} escalations - Collects the reason of every escalation delivered to this process.
429
- * @returns {import("../src/managed-process.js").default} A process whose retirement wait already expired.
430
- */
431
- function wedgedProcess(pid, escalations) {
432
- // eslint-disable-next-line local/no-unknown-jsdoc-type
433
- return /** @type {import("../src/managed-process.js").default} */ (/** @type {unknown} */ ({
434
- escalateRetirement: async (/** @type {{reason: string}} */ options) => {
435
- escalations.push(options.reason)
436
-
437
- return true
438
- },
439
- quiesceStrict: async () => {},
440
- retirementTimedOut: true,
441
- status: () => ({pid, retirementTimedOut: true, state: "quiesced"}),
442
- stop: async () => {}
443
- }))
444
- }
445
-
446
- test("the supervisor escalates a wedged generation whose residue holds no runner child", async () => {
447
- const procPath = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-escalate-wedged-"))
448
-
449
- try {
450
- // Group leader only, 5000ms of consumed CPU: nothing left in the group but the wedged worker.
451
- writeProcMember(procPath, 101, 101, "S", 300, 200)
452
-
453
- /** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
454
- const logs = []
455
- /** @type {string[]} */
456
- const escalations = []
457
- const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}}, () => true, procPath)
458
-
459
- release.logger = (message, data) => logs.push({data, message})
460
- release.state = "stopped"
461
- release.processes.set("worker", wedgedProcess(101, escalations))
462
- // The generation already recorded `release drained`; this is the observation its retirement
463
- // left behind, a full window behind the read that now acts on it.
464
- release.drainNewestSample = {
465
- atMs: Date.now() - 25000,
466
- processes: [{cpuMs: 5000, heldProcessGroupPgid: undefined, id: "worker", liveChildCount: 0, pid: 101, retirementTimedOut: true}]
467
- }
468
-
469
- const escalated = await release.escalateWedgedGeneration({reason: "deploy of v2"})
470
-
471
- expect(escalated).toEqual(["worker"])
472
- expect(escalations).toHaveLength(1)
473
- expect(escalations[0]).toContain("deploy of v2")
474
- expect(escalations[0]).toContain("no runner child remaining")
475
- expect(logs).toMatchObject([{data: {escalation: "SIGKILL", processIds: ["worker"], releaseId: "v1", trigger: "deploy of v2"}, message: "release wedged generation escalated"}])
476
- } finally {
477
- fs.rmSync(procPath, {force: true, recursive: true})
478
- }
479
- })
480
-
481
- test("a wedged generation that still holds a live runner child is reported, never killed", async () => {
482
- const procPath = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-escalate-refused-"))
483
-
484
- try {
485
- writeProcMember(procPath, 101, 101, "S", 300, 200)
486
- // A live runner child: the wedge is real, and killing it is still the STOE-1077 failure.
487
- writeProcMember(procPath, 102, 101, "S", 0, 0)
488
-
489
- /** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
490
- const logs = []
491
- /** @type {string[]} */
492
- const escalations = []
493
- const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}}, () => true, procPath)
494
-
495
- release.logger = (message, data) => logs.push({data, message})
496
- release.state = "stopped"
497
- release.processes.set("worker", wedgedProcess(101, escalations))
498
- release.drainNewestSample = {
499
- atMs: Date.now() - 25000,
500
- processes: [{cpuMs: 5000, heldProcessGroupPgid: undefined, id: "worker", liveChildCount: 1, pid: 101, retirementTimedOut: true}]
501
- }
502
-
503
- const escalated = await release.escalateWedgedGeneration({reason: "deploy of v2"})
504
-
505
- expect(escalated).toEqual([])
506
- expect(escalations).toEqual([])
507
- expect(logs).toMatchObject([{data: {escalation: "refused", pids: [101], processIds: ["worker"], releaseId: "v1"}, message: "release wedged generation left alive"}])
508
- expect(logs[0].data?.reason).toBe("no escalation: 1 runner child is still live in the generation; SIGKILL would abort in-flight work")
509
- } finally {
510
- fs.rmSync(procPath, {force: true, recursive: true})
511
- }
512
- })
513
-
514
- test("the retirement transition brackets its own wait and reports a residue it cannot yet prove wedged", async () => {
515
- const procPath = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-escalate-bracket-"))
516
-
517
- try {
518
- writeProcMember(procPath, 4242, 4242, "S", 300, 200)
519
-
520
- /** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
521
- const logs = []
522
- /** @type {string[]} */
523
- const escalations = []
524
- const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}}, () => true, procPath)
525
-
526
- release.logger = (message, data) => logs.push({data, message})
527
- release.state = "draining"
528
- release.nonBlockingDrainIds.add("worker")
529
- release.handoffServiceIds.add("jobs-main")
530
- release.processes.set("worker", wedgedProcess(4242, escalations))
531
- // eslint-disable-next-line local/no-unknown-jsdoc-type
532
- release.processes.set("jobs-main", /** @type {import("../src/managed-process.js").default} */ (/** @type {unknown} */ ({
533
- status: () => ({pid: undefined, retirementTimedOut: false, state: "stopped"}),
534
- stop: async () => {}
535
- })))
536
-
537
- await release.drainAndStop(0, {
538
- ...release.config,
539
- proxy: {...release.config.proxy, nonBlockingDrainTimeoutMs: 20},
540
- releaseRetention: {...release.config.releaseRetention, escalateWedgedGenerations: true}
541
- })
542
-
543
- // The transition still completes: the escalation is its last act, never a wait of its own.
544
- expect(release.state).toBe("stopped")
545
- expect(escalations).toEqual([])
546
- expect(release.drainPriorSample).toBeDefined()
547
- expect(release.drainNewestSample).toBeDefined()
548
- expect(logs).toMatchObject([
549
- {data: {pid: 4242, processId: "worker"}, message: "release non-blocking process still draining"},
550
- {data: {escalation: "refused", pids: [4242], releaseId: "v1", trigger: "release retirement"}, message: "release wedged generation left alive"}
551
- ])
552
- // A bracket shorter than the flatness window cannot justify a kill; the retained pair is
553
- // what lets the next deploy prove the residue without sampling on a timer.
554
- expect(logs[1].data?.reason).toContain("is below the 20000ms minimum")
555
- } finally {
556
- fs.rmSync(procPath, {force: true, recursive: true})
557
- }
558
- })
559
153
  })