rollbridge 0.1.66 → 0.1.67

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,7 +2,7 @@
2
2
 
3
3
  import {spawn} from "node:child_process"
4
4
  import {EventEmitter} from "node:events"
5
- import {classifyDrainProgress} from "./drain-policy.js"
5
+ import {classifyDrainProgress, decideWedgedEscalation} from "./drain-policy.js"
6
6
  import ManagedProcess from "./managed-process.js"
7
7
  import {findAvailablePort} from "./port-allocator.js"
8
8
  import {processStartOrder} from "./process-order.js"
@@ -863,6 +863,15 @@ export default class ReleaseGroup extends EventEmitter {
863
863
  }))
864
864
  const handoffServices = entries.filter(([id]) => this.handoffServiceIds.has(id)).map(([, processInstance]) => processInstance)
865
865
  const connectionDependent = entries.filter(([id]) => !this.nonBlockingDrainIds.has(id) && !this.handoffServiceIds.has(id)).map(([, processInstance]) => processInstance)
866
+ // Bracket the retirement wait with a pair of observations: this one, and the one taken when
867
+ // every stop has returned. The gap between them is the wait the transition already performs
868
+ // (nonBlockingDrainTimeoutMs at minimum), so CPU flatness can be proven from the transition
869
+ // itself rather than from a sampler running alongside it.
870
+ const retirementBaselineSample = config.releaseRetention.escalateWedgedGenerations && nonBlockingStops.length > 0
871
+ ? this.takeLivenessSample(Date.now())
872
+ : undefined
873
+
874
+ if (retirementBaselineSample) this.drainNewestSample = retirementBaselineSample
866
875
 
867
876
  if (this.connectionCount > 0) {
868
877
  await new Promise((resolve) => {
@@ -899,11 +908,21 @@ export default class ReleaseGroup extends EventEmitter {
899
908
  }
900
909
  })
901
910
  await Promise.allSettled(handoffServices.map((processInstance) => processInstance.stop()))
911
+
912
+ // Last act of the retirement transition, before the release is recorded stopped: whatever
913
+ // is still alive here is the residue, and the bracket pair above can now say whether it is
914
+ // wedged. A generation that stops with a live process group is the state that made
915
+ // operators signal PIDs by hand (STOE-1687), so the transition decides it or reports it —
916
+ // it does not end silently with the process still holding the release.
917
+ if (retirementBaselineSample) await this.escalateWedgedGeneration({previousSample: retirementBaselineSample, reason: "release retirement"})
918
+
902
919
  this.state = "stopped"
903
920
  this.stoppedAt = new Date().toISOString()
904
921
  this.releasePortReservations()
905
922
  this.abandonLocalConnections()
906
- this.clearDrainLivenessSamples()
923
+ // The bracket pair is deliberately kept: a generation that stopped with a live process
924
+ // group stays observable, and the deploy path needs the older observation to prove the
925
+ // residue is still flat rather than starting a fresh window from zero.
907
926
  }
908
927
 
909
928
  /**
@@ -938,11 +957,39 @@ export default class ReleaseGroup extends EventEmitter {
938
957
  this.retirementError = undefined
939
958
  }
940
959
 
941
- /** @returns {Promise<void>} Stops all release-owned processes. */
942
- async stop() {
943
- const stopTasks = [...this.processes.values()].map((processInstance) => processInstance.stop())
960
+ /**
961
+ * Stops every process owned by the generation.
962
+ * @param {{retirementTimeoutMs?: number}} [options] - Retirement budget for the processes the
963
+ * supervisor must not kill: a non-blocking-drain process, and any process whose graceful window
964
+ * is `"indefinite"`. Without it those stops have no deadline at all, so an explicit
965
+ * `rollbridge stop` can wait forever — and because that command runs inside an owner mutation,
966
+ * the wait holds the mutation fence and every later deploy is refused (STOE-2015).
967
+ * @returns {Promise<void>} Resolves when stopped, or when retirement stops waiting.
968
+ */
969
+ async stop(options = {}) {
970
+ const {retirementTimeoutMs} = options
971
+ /** @type {{bounded: boolean, id: string, instance: ManagedProcess, promise: Promise<void>}[]} */
972
+ const stopTasks = [...this.processes.entries()].map(([id, processInstance]) => {
973
+ const bounded = retirementTimeoutMs !== undefined && (this.nonBlockingDrainIds.has(id) || processInstance.hasUnboundedGracefulWindow())
974
+
975
+ return {bounded, id, instance: processInstance, promise: processInstance.stop(bounded ? {retirementTimeoutMs} : {})}
976
+ })
977
+
978
+ await Promise.allSettled(stopTasks.map(({promise}) => promise))
979
+
980
+ // A stop that stopped waiting leaves the process running. Name it, so the operator sees the
981
+ // residue instead of a command that produced nothing.
982
+ for (const {bounded, id, instance} of stopTasks) {
983
+ if (!bounded || !instance.retirementTimedOut) continue
984
+
985
+ this.logger("release stop gave up waiting for a retiring process", {
986
+ pid: instance.status().pid ?? null,
987
+ processId: id,
988
+ releaseId: this.releaseId,
989
+ timeoutMs: retirementTimeoutMs
990
+ })
991
+ }
944
992
 
945
- await Promise.allSettled(stopTasks)
946
993
  this.state = "stopped"
947
994
  this.stoppedAt = new Date().toISOString()
948
995
  this.releasePortReservations()
@@ -969,32 +1016,109 @@ export default class ReleaseGroup extends EventEmitter {
969
1016
  drainStatus() {
970
1017
  if (this.state !== "draining") return undefined
971
1018
 
972
- const nowMs = Date.now()
1019
+ return this.observeRetirement({nowMs: Date.now()}).classification
1020
+ }
973
1021
 
974
- if (this.connectionCount > 0) {
975
- return classifyDrainProgress({
976
- connectionCount: this.connectionCount,
977
- currentSample: this.drainNewestSample ?? {atMs: nowMs, processes: []},
978
- minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
979
- previousSample: undefined
980
- })
1022
+ /**
1023
+ * Records the newest liveness observation of this generation and classifies its retirement.
1024
+ * @param {object} options - Observation options.
1025
+ * @param {number} options.nowMs - Read time.
1026
+ * @param {DrainLivenessSample | undefined} [options.previousSample] - Explicit older observation to classify against. The retirement transition brackets its own wait with one instead of reusing a stored pair.
1027
+ * @returns {{classification: ReleaseDrainStatus, currentSample: DrainLivenessSample}} The observation the decision acts on, and what the pair says.
1028
+ */
1029
+ observeRetirement({nowMs, previousSample}) {
1030
+ if (previousSample === undefined && this.connectionCount > 0) {
1031
+ const currentSample = this.drainNewestSample ?? {atMs: nowMs, processes: []}
1032
+
1033
+ return {
1034
+ classification: classifyDrainProgress({
1035
+ connectionCount: this.connectionCount,
1036
+ currentSample,
1037
+ minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
1038
+ previousSample: undefined
1039
+ }),
1040
+ currentSample
1041
+ }
981
1042
  }
982
1043
 
983
1044
  const newest = this.drainNewestSample
984
- const currentSample = newest && nowMs - newest.atMs < DRAIN_SAMPLE_WINDOW_MS ? newest : this.takeLivenessSample(nowMs)
985
- const previousSample = currentSample === newest ? this.drainPriorSample : newest
1045
+ const currentSample = previousSample !== undefined || newest === undefined || nowMs - newest.atMs >= DRAIN_SAMPLE_WINDOW_MS
1046
+ ? this.takeLivenessSample(nowMs)
1047
+ : newest
1048
+ const priorSample = previousSample ?? (currentSample === newest ? this.drainPriorSample : newest)
986
1049
 
987
1050
  if (currentSample !== newest) {
988
1051
  this.drainPriorSample = newest
989
1052
  this.drainNewestSample = currentSample
990
1053
  }
991
1054
 
992
- return classifyDrainProgress({
993
- connectionCount: this.connectionCount,
994
- currentSample,
995
- minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
996
- previousSample
997
- })
1055
+ return {
1056
+ classification: classifyDrainProgress({
1057
+ connectionCount: this.connectionCount,
1058
+ currentSample,
1059
+ minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
1060
+ previousSample: priorSample
1061
+ }),
1062
+ currentSample
1063
+ }
1064
+ }
1065
+
1066
+ /**
1067
+ * Applies the wedge escalation to this generation: terminates the processes a proven wedge
1068
+ * proves are holding nothing, and names the ones it refused to terminate.
1069
+ *
1070
+ * Both halves are required. The card this path was written for (STOE-1687) is about a
1071
+ * wedged worker holding a release open for four hours while deploys stack behind it; the
1072
+ * boundary on the same card (STOE-1077) is that no process still holding live work may ever
1073
+ * be SIGKILLed. So a wedge that is not kill-safe is not silently skipped — it is reported
1074
+ * with the pids and process groups it is holding, which is the information an operator needs
1075
+ * in place of guessing.
1076
+ * @param {object} options - Escalation context.
1077
+ * @param {DrainLivenessSample | undefined} [options.previousSample] - Older observation bracketing a wait this caller already performed.
1078
+ * @param {string} options.reason - Named context recorded with the decision.
1079
+ * @returns {Promise<string[]>} Instance ids of the processes the SIGKILL was delivered to.
1080
+ */
1081
+ async escalateWedgedGeneration({previousSample, reason}) {
1082
+ const {classification, currentSample} = this.observeRetirement({nowMs: Date.now(), previousSample})
1083
+ const decision = decideWedgedEscalation({classification, currentSample, minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS})
1084
+ const liveObservations = currentSample.processes.filter((process) => process.pid !== undefined || process.heldProcessGroupPgid !== undefined)
1085
+
1086
+ if (decision.decision === "report") {
1087
+ if (classification.status === "wedged") {
1088
+ this.logger("release wedged generation left alive", {
1089
+ escalation: "refused",
1090
+ heldProcessGroupPgids: liveObservations.map((process) => process.heldProcessGroupPgid ?? null),
1091
+ pids: liveObservations.map((process) => process.pid ?? null),
1092
+ processIds: liveObservations.map((process) => process.id),
1093
+ reason: decision.reason,
1094
+ releaseId: this.releaseId,
1095
+ trigger: reason
1096
+ })
1097
+ }
1098
+
1099
+ return []
1100
+ }
1101
+
1102
+ const escalated = []
1103
+
1104
+ for (const target of decision.targets) {
1105
+ const processInstance = this.processes.get(target.processId)
1106
+
1107
+ if (!processInstance) continue
1108
+ if (await processInstance.escalateRetirement({reason: `${reason}: ${decision.reason}`})) escalated.push(target.processId)
1109
+ }
1110
+
1111
+ if (escalated.length > 0) {
1112
+ this.logger("release wedged generation escalated", {
1113
+ escalation: "SIGKILL",
1114
+ processIds: escalated,
1115
+ reason: decision.reason,
1116
+ releaseId: this.releaseId,
1117
+ trigger: reason
1118
+ })
1119
+ }
1120
+
1121
+ return escalated
998
1122
  }
999
1123
 
1000
1124
  /**
@@ -705,12 +705,12 @@ test("validateConfig defaults releaseRetention, accepts overrides, and rejects b
705
705
  const defaulted = validateRetention(undefined)
706
706
 
707
707
  expect(defaulted.issues).toEqual([])
708
- expect(defaulted.config.releaseRetention).toEqual({keep: 10, maxAgeMs: 0, maxConcurrentDrainingReleaseGroups: 12})
708
+ expect(defaulted.config.releaseRetention).toEqual({escalateWedgedGenerations: false, keep: 10, maxAgeMs: 0, maxConcurrentDrainingReleaseGroups: 12})
709
709
 
710
- const custom = validateRetention({keep: 3, maxAgeMs: 60000, maxConcurrentDrainingReleaseGroups: 4})
710
+ const custom = validateRetention({escalateWedgedGenerations: true, keep: 3, maxAgeMs: 60000, maxConcurrentDrainingReleaseGroups: 4})
711
711
 
712
712
  expect(custom.issues).toEqual([])
713
- expect(custom.config.releaseRetention).toEqual({keep: 3, maxAgeMs: 60000, maxConcurrentDrainingReleaseGroups: 4})
713
+ expect(custom.config.releaseRetention).toEqual({escalateWedgedGenerations: true, keep: 3, maxAgeMs: 60000, maxConcurrentDrainingReleaseGroups: 4})
714
714
 
715
715
  // 0 is a valid bound: it disables the pre-activation draining-generation check.
716
716
  const unbounded = validateRetention({maxConcurrentDrainingReleaseGroups: 0})
@@ -718,8 +718,9 @@ test("validateConfig defaults releaseRetention, accepts overrides, and rejects b
718
718
  expect(unbounded.issues).toEqual([])
719
719
  expect(unbounded.config.releaseRetention.maxConcurrentDrainingReleaseGroups).toBe(0)
720
720
 
721
- const invalid = validateRetention({keep: -1, maxAgeMs: -5, maxConcurrentDrainingReleaseGroups: -2})
721
+ const invalid = validateRetention({escalateWedgedGenerations: "yes", keep: -1, maxAgeMs: -5, maxConcurrentDrainingReleaseGroups: -2})
722
722
 
723
+ expect(invalid.issues.some((issue) => issue.message === "releaseRetention.escalateWedgedGenerations must be a boolean")).toBeTruthy()
723
724
  expect(invalid.issues.some((issue) => issue.message === "releaseRetention.keep must be a non-negative integer")).toBeTruthy()
724
725
  expect(invalid.issues.some((issue) => issue.message === "releaseRetention.maxAgeMs must be a non-negative number")).toBeTruthy()
725
726
  expect(invalid.issues.some((issue) => issue.message === "releaseRetention.maxConcurrentDrainingReleaseGroups must be a non-negative integer")).toBeTruthy()
@@ -1,7 +1,7 @@
1
1
  // @ts-check
2
2
 
3
3
  import {describe, expect, test} from "@velocious/testing"
4
- import {admitGenerationRelease, classifyDrainProgress} from "../src/drain-policy.js"
4
+ import {admitGenerationRelease, classifyDrainProgress, decideWedgedEscalation} from "../src/drain-policy.js"
5
5
 
6
6
  describe("drain-policy", () => {
7
7
 
@@ -47,6 +47,23 @@ function classify(options = {}) {
47
47
  })
48
48
  }
49
49
 
50
+ /**
51
+ * @param {object} [options] - Decision inputs.
52
+ * @param {number} [options.connectionCount] - Live connections owned by the generation.
53
+ * @param {import("../src/drain-policy.js").DrainLivenessSample} [options.current] - Newest observation.
54
+ * @param {import("../src/drain-policy.js").DrainLivenessSample | undefined} [options.previous] - Older observation.
55
+ * @returns {import("../src/drain-policy.js").EscalationDecision} Escalation decision for the same pair the classifier saw.
56
+ */
57
+ function escalate(options = {}) {
58
+ const currentSample = options.current ?? sample()
59
+
60
+ return decideWedgedEscalation({
61
+ classification: classify({connectionCount: options.connectionCount, current: currentSample, previous: options.previous}),
62
+ currentSample,
63
+ minSampleWindowMs: WINDOW_MS
64
+ })
65
+ }
66
+
50
67
  test("a generation that still owns connections is draining, never wedged", () => {
51
68
  const result = classify({connectionCount: 3, current: sample({liveChildCount: 0})})
52
69
 
@@ -151,6 +168,90 @@ test("a held process group that still consumes CPU across the window is a progre
151
168
  expect(result.reason).toBe("CPU advanced 4000ms across 25000ms")
152
169
  })
153
170
 
171
+ test("a proven wedge with no runner child left is escalated, naming the process group it terminates", () => {
172
+ const previous = sample({atMs: 1000000, cpuMs: 5000, liveChildCount: 3})
173
+ const current = sample({atMs: 1025000, cpuMs: 5000, liveChildCount: 0})
174
+ const decision = escalate({current, previous})
175
+
176
+ expect(decision.decision).toBe("escalate")
177
+ expect(decision.killSafe).toBe(true)
178
+ expect(decision.targets).toEqual([{pgid: 4242, processId: "worker"}])
179
+ expect(decision.reason).toContain("CPU flat across 25000ms")
180
+ })
181
+
182
+ test("the escalation targets the group that outlived its tracked process", () => {
183
+ const previous = sample({atMs: 1000000, cpuMs: 5000, heldProcessGroupPgid: 4242, liveChildCount: 2, pid: undefined})
184
+ const current = sample({atMs: 1025000, cpuMs: 5000, heldProcessGroupPgid: 4242, liveChildCount: 0, pid: undefined})
185
+ const decision = escalate({current, previous})
186
+
187
+ expect(decision.decision).toBe("escalate")
188
+ expect(decision.targets).toEqual([{pgid: 4242, processId: "worker"}])
189
+ })
190
+
191
+ test("a wedge that still holds a live runner child is never escalated", () => {
192
+ const previous = sample({atMs: 1000000, cpuMs: 5000})
193
+ const current = sample({atMs: 1025000, cpuMs: 5000, liveChildCount: 3})
194
+
195
+ expect(classify({current, previous}).status).toBe("wedged")
196
+
197
+ const decision = escalate({current, previous})
198
+
199
+ expect(decision.decision).toBe("report")
200
+ expect(decision.killSafe).toBe(false)
201
+ expect(decision.targets).toEqual([])
202
+ expect(decision.reason).toBe("no escalation: 3 runner children are still live in the generation; SIGKILL would abort in-flight work")
203
+ })
204
+
205
+ test("a live process whose CPU is still moving is never escalated", () => {
206
+ const previous = sample({atMs: 1000000, cpuMs: 5000})
207
+ const current = sample({atMs: 1025000, cpuMs: 9000, liveChildCount: 0})
208
+
209
+ expect(classify({current, previous}).status).toBe("wedged")
210
+
211
+ const decision = escalate({current, previous})
212
+
213
+ expect(decision.decision).toBe("report")
214
+ expect(decision.reason).toBe("no escalation: CPU flatness is not proven (4000ms of CPU moved across 25000ms)")
215
+ })
216
+
217
+ test("a process that has not expired its retirement budget is never escalated", () => {
218
+ const previous = sample({atMs: 1000000, cpuMs: 5000, liveChildCount: 0, retirementTimedOut: false})
219
+ const current = sample({atMs: 1025000, cpuMs: 5000, liveChildCount: 0, retirementTimedOut: false})
220
+ const decision = escalate({current, previous})
221
+
222
+ expect(decision.decision).toBe("report")
223
+ expect(decision.reason).toBe("no escalation: drain is progressing (process worker is still inside its retirement budget)")
224
+ })
225
+
226
+ test("a generation that still owns a live connection is never escalated", () => {
227
+ const decision = escalate({connectionCount: 2, current: sample({liveChildCount: 0})})
228
+
229
+ expect(decision.decision).toBe("report")
230
+ expect(decision.reason).toBe("no escalation: drain is progressing (2 live connections still owned by the generation)")
231
+ })
232
+
233
+ test("a wedge with no measurable CPU is never escalated", () => {
234
+ const previous = sample({atMs: 1000000, cpuMs: undefined, liveChildCount: 0})
235
+ const current = sample({atMs: 1025000, cpuMs: undefined, liveChildCount: 0})
236
+
237
+ expect(classify({current, previous}).status).toBe("wedged")
238
+
239
+ const decision = escalate({current, previous})
240
+
241
+ expect(decision.decision).toBe("report")
242
+ expect(decision.reason).toBe("no escalation: CPU flatness is not proven (process CPU is not measurable on this host)")
243
+ })
244
+
245
+ test("a wedge seen in one observation is not escalated until a full window has passed", () => {
246
+ const current = sample({liveChildCount: 0})
247
+ const shortWindow = escalate({current: sample({atMs: 1001000, cpuMs: 5000, liveChildCount: 0}), previous: sample({atMs: 1000000, cpuMs: 5000, liveChildCount: 0})})
248
+
249
+ expect(classify({current}).status).toBe("wedged")
250
+ expect(escalate({current}).decision).toBe("report")
251
+ expect(escalate({current}).reason).toBe("no escalation: liveness window 0ms is below the 20000ms minimum")
252
+ expect(shortWindow.reason).toBe("no escalation: liveness window 1000ms is below the 20000ms minimum")
253
+ })
254
+
154
255
  test("admission allows a handoff that stays within the bound and counts the incumbent", () => {
155
256
  const admission = admitGenerationRelease({
156
257
  candidateReleaseId: "v3",
@@ -309,10 +309,12 @@ test("runs quiet and drain lifecycle hooks before stopping", async () => {
309
309
  })
310
310
 
311
311
  test("retirement timeout leaves an indefinite worker alive without SIGKILL", async () => {
312
+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-retirement-wedge-"))
313
+ const readyPath = path.join(dir, "ready")
312
314
  /** @type {string[]} */
313
315
  const signals = []
314
316
  const managed = new ManagedProcess({
315
- command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify("process.on('SIGTERM', () => {}); setInterval(() => {}, 1000)")}`,
317
+ command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`process.on('SIGTERM', () => {}); require("node:fs").writeFileSync(${JSON.stringify(readyPath)}, "ready"); setInterval(() => {}, 1000)`)}`,
316
318
  cwd: undefined,
317
319
  env: {},
318
320
  id: "worker",
@@ -332,8 +334,210 @@ test("retirement timeout leaves an indefinite worker alive without SIGKILL", asy
332
334
 
333
335
  try {
334
336
  await managed.start()
337
+ // The fixture installs its SIGTERM disposition before it announces readiness. Signalling a
338
+ // child that has not finished booting reaches the default disposition instead, which exits
339
+ // the group and leaves no wedge to observe.
340
+ await waitFor(() => fs.existsSync(readyPath))
341
+ await managed.stop({retirementTimeoutMs: 25})
342
+
343
+ expect(managed.retirementTimedOut).toBe(true)
344
+ expect(signals).toEqual(["SIGTERM"])
345
+ expect(managed.status().pid).toBeDefined()
346
+ } finally {
347
+ const pid = managed.status().pid
348
+ if (pid) process.kill(-pid, "SIGKILL")
349
+ if (managed.exitPromise) await managed.exitPromise
350
+ fs.rmSync(dir, {force: true, recursive: true})
351
+ }
352
+ })
353
+
354
+ test("escalateRetirement terminates the wedged worker the bounded retirement left alive", async () => {
355
+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-escalation-wedge-"))
356
+ const readyPath = path.join(dir, "ready")
357
+ /** @type {string[]} */
358
+ const signals = []
359
+ /** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
360
+ const logs = []
361
+ const managed = new ManagedProcess({
362
+ command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`process.on('SIGTERM', () => {}); require("node:fs").writeFileSync(${JSON.stringify(readyPath)}, "ready"); setInterval(() => {}, 1000)`)}`,
363
+ cwd: undefined,
364
+ env: {},
365
+ id: "worker",
366
+ lifecycle: {drainTimeoutMs: 0},
367
+ logger: (message, data) => logs.push({data, message}),
368
+ outputLines: 50,
369
+ restartDelayMs: 10,
370
+ shouldRestart: () => false,
371
+ stopSignal: "SIGTERM",
372
+ stopTimeoutMs: "indefinite"
373
+ })
374
+ const signalProcessGroup = managed.signalProcessGroup.bind(managed)
375
+
376
+ managed.signalProcessGroup = async (signal, pgid, deadline) => {
377
+ signals.push(signal)
378
+ await signalProcessGroup(signal, pgid, deadline)
379
+ }
380
+
381
+ try {
382
+ await managed.start()
383
+ await waitFor(() => fs.existsSync(readyPath))
335
384
  await managed.stop({retirementTimeoutMs: 25})
385
+ const wedgedPid = managed.status().pid
386
+
387
+ expect(managed.retirementTimedOut).toBe(true)
388
+ expect(signals).toEqual(["SIGTERM"])
389
+ expect(wedgedPid).toBeDefined()
390
+
391
+ expect(await managed.escalateRetirement({reason: "spec: proven wedge"})).toBe(true)
392
+ expect(signals).toEqual(["SIGTERM", "SIGKILL"])
393
+ expect(logs.some((entry) => entry.message === "process retirement escalated to SIGKILL" && entry.data?.reason === "spec: proven wedge")).toBe(true)
394
+ expect(await managed.waitForProcessGroupExit(/** @type {number} */ (wedgedPid), Date.now() + 5000)).toBe(true)
395
+ if (managed.exitPromise) await managed.exitPromise
396
+ expect(managed.status().pid).toBeUndefined()
397
+ expect(managed.status().state).toBe("stopped")
398
+ } finally {
399
+ const pid = managed.status().pid
400
+ if (pid) process.kill(-pid, "SIGKILL")
401
+ if (managed.exitPromise) await managed.exitPromise
402
+ fs.rmSync(dir, {force: true, recursive: true})
403
+ }
404
+ })
405
+
406
+ test("escalateRetirement refuses a process that has not expired its retirement budget", async () => {
407
+ /** @type {string[]} */
408
+ const signals = []
409
+ /** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
410
+ const logs = []
411
+ const managed = new ManagedProcess({
412
+ command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify("setInterval(() => {}, 1000)")}`,
413
+ cwd: undefined,
414
+ env: {},
415
+ id: "worker",
416
+ lifecycle: {drainTimeoutMs: 0},
417
+ logger: (message, data) => logs.push({data, message}),
418
+ outputLines: 50,
419
+ restartDelayMs: 10,
420
+ shouldRestart: () => false,
421
+ stopSignal: "SIGTERM",
422
+ stopTimeoutMs: "indefinite"
423
+ })
424
+ const signalProcessGroup = managed.signalProcessGroup.bind(managed)
336
425
 
426
+ managed.signalProcessGroup = async (signal, pgid, deadline) => {
427
+ signals.push(signal)
428
+ await signalProcessGroup(signal, pgid, deadline)
429
+ }
430
+
431
+ try {
432
+ await managed.start()
433
+ await managed.quiesce()
434
+
435
+ expect(managed.retirementTimedOut).toBe(false)
436
+ expect(await managed.escalateRetirement({reason: "spec"})).toBe(false)
437
+ expect(signals).toEqual([])
438
+ expect(logs.some((entry) => entry.message === "retirement escalation refused; process is not past its retirement budget")).toBe(true)
439
+ expect(managed.status().pid).toBeDefined()
440
+ } finally {
441
+ const pid = managed.status().pid
442
+ if (pid) process.kill(-pid, "SIGKILL")
443
+ if (managed.exitPromise) await managed.exitPromise
444
+ }
445
+ })
446
+
447
+ test("escalateRetirement is inert for a process that drained normally", async () => {
448
+ /** @type {string[]} */
449
+ const signals = []
450
+ const managed = new ManagedProcess({
451
+ command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify("setInterval(() => {}, 1000)")}`,
452
+ cwd: undefined,
453
+ env: {},
454
+ id: "worker",
455
+ lifecycle: {drainTimeoutMs: 0},
456
+ logger: () => {},
457
+ outputLines: 50,
458
+ restartDelayMs: 10,
459
+ shouldRestart: () => false,
460
+ stopSignal: "SIGTERM",
461
+ stopTimeoutMs: 2000
462
+ })
463
+ const signalProcessGroup = managed.signalProcessGroup.bind(managed)
464
+
465
+ managed.signalProcessGroup = async (signal, pgid, deadline) => {
466
+ signals.push(signal)
467
+ await signalProcessGroup(signal, pgid, deadline)
468
+ }
469
+
470
+ try {
471
+ await managed.start()
472
+ await managed.stop()
473
+
474
+ expect(managed.status().state).toBe("stopped")
475
+ expect(await managed.escalateRetirement({reason: "spec"})).toBe(false)
476
+ expect(signals).toEqual(["SIGTERM"])
477
+ } finally {
478
+ const pid = managed.status().pid
479
+ if (pid) process.kill(-pid, "SIGKILL")
480
+ if (managed.exitPromise) await managed.exitPromise
481
+ }
482
+ })
483
+
484
+ test("a retirement budget caps a finite graceful window instead of being outvoted by it", async () => {
485
+ /** @type {string[]} */
486
+ const signals = []
487
+ /** @type {Array<{msFromStopStart: number, phase: string}>} */
488
+ const deadlines = []
489
+ const managed = new ManagedProcess({
490
+ command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify("process.on('SIGTERM', () => {}); setInterval(() => {}, 1000)")}`,
491
+ cwd: undefined,
492
+ env: {},
493
+ id: "worker",
494
+ lifecycle: {drainTimeoutMs: 0},
495
+ logger: () => {},
496
+ outputLines: 50,
497
+ restartDelayMs: 10,
498
+ shouldRestart: () => false,
499
+ stopSignal: "SIGTERM",
500
+ stopTimeoutMs: 2000
501
+ })
502
+ let stopStartedAtMs = 0
503
+
504
+ /**
505
+ * @param {string} phase - The step the deadline was handed to.
506
+ * @param {number | undefined} deadline - Absolute deadline the step received.
507
+ * @returns {void} Records the deadline as an offset from the start of the stop.
508
+ */
509
+ const recordDeadline = (phase, deadline) => {
510
+ if (deadline !== undefined) deadlines.push({msFromStopStart: deadline - stopStartedAtMs, phase})
511
+ }
512
+
513
+ const signalProcessGroup = managed.signalProcessGroup.bind(managed)
514
+ managed.signalProcessGroup = async (signal, pgid, deadline) => {
515
+ signals.push(signal)
516
+ recordDeadline("signal", deadline)
517
+ await signalProcessGroup(signal, pgid, deadline)
518
+ }
519
+
520
+ const waitForProcessGroupExit = managed.waitForProcessGroupExit.bind(managed)
521
+ managed.waitForProcessGroupExit = async (pgid, deadline) => {
522
+ recordDeadline("wait", deadline)
523
+ return await waitForProcessGroupExit(pgid, deadline)
524
+ }
525
+
526
+ try {
527
+ await managed.start()
528
+ stopStartedAtMs = Date.now()
529
+
530
+ await managed.stop({retirementTimeoutMs: 25})
531
+
532
+ // The 2 000 ms graceful window must not outvote the 25 ms retirement budget. Read the other
533
+ // way round the wait runs the whole window before it escalates to SIGKILL, which is the shape
534
+ // that held the TensorBuzz control plane for 4,5 h after the worker moved from "indefinite" to
535
+ // 4,5 h and killed the in-flight builds the consumer said not to interrupt.
536
+ expect(deadlines.map((entry) => entry.phase)).toEqual(["signal", "wait"])
537
+ for (const entry of deadlines) {
538
+ expect(entry.msFromStopStart).toBeGreaterThanOrEqual(25)
539
+ expect(entry.msFromStopStart).toBeLessThan(1000)
540
+ }
337
541
  expect(managed.retirementTimedOut).toBe(true)
338
542
  expect(signals).toEqual(["SIGTERM"])
339
543
  expect(managed.status().pid).toBeDefined()