rollbridge 0.1.65 → 0.1.67

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -9,6 +9,10 @@ const MAX_BUFFERED_OUTPUT_CHARACTERS = 64 * 1024
9
9
  // Reading a process group means scanning procfs, and status() is read continuously. A held
10
10
  // group is re-verified on this interval instead of on every read.
11
11
  const HELD_GROUP_RECHECK_MS = 5000
12
+ // How long an escalated SIGKILL is given to clear the group before the group is recorded as
13
+ // outliving it again. Bounded so the escalation itself can never become the unbounded wait it
14
+ // replaced.
15
+ const ESCALATION_KILL_CONFIRM_MS = 5000
12
16
 
13
17
  /**
14
18
  * @typedef {import("./json.js").JsonValue} JsonValue
@@ -525,20 +529,25 @@ export default class ManagedProcess extends EventEmitter {
525
529
  if (this.processGroupExists(pgid)) {
526
530
  const timeoutMs = options.timeoutMs ?? this.stopTimeoutMs
527
531
  const retirementDeadline = options.retirementTimeoutMs === undefined ? undefined : Date.now() + options.retirementTimeoutMs
528
- let gracefulDeadline
532
+ const gracefulDeadline = timeoutMs === "indefinite" ? undefined : Date.now() + timeoutMs
533
+ // A retirement budget is the caller's decision that this process must not be waited for and
534
+ // must not be killed, so it caps the graceful window rather than only standing in for a
535
+ // missing one. Read the other way round, a finite `gracefulStopMs` outvotes the retirement
536
+ // bound and the wait runs for the whole window: TensorBuzz production moved its worker from
537
+ // `"indefinite"` to 4,5 h and the stop would still have held the control plane for 4,5 h.
538
+ const retirementBound = retirementDeadline !== undefined && (gracefulDeadline === undefined || retirementDeadline < gracefulDeadline)
539
+ const waitDeadline = retirementBound ? retirementDeadline : gracefulDeadline
529
540
 
530
541
  if (stopCommand) {
531
542
  await this.runHook(stopCommand, hookTimeoutMs, "stop command", pgid)
532
- gracefulDeadline = timeoutMs === "indefinite" ? undefined : Date.now() + timeoutMs
533
543
  } else {
534
544
  this.intentionalStopSignal = /** @type {ProcessExitSignal} */ (this.stopSignal)
535
- gracefulDeadline = timeoutMs === "indefinite" ? undefined : Date.now() + timeoutMs
536
- await this.signalProcessGroup(this.stopSignal, pgid, gracefulDeadline ?? retirementDeadline)
545
+ await this.signalProcessGroup(this.stopSignal, pgid, waitDeadline)
537
546
  }
538
547
 
539
- const exited = await this.waitForProcessGroupExit(pgid, gracefulDeadline === undefined ? retirementDeadline : gracefulDeadline)
548
+ const exited = await this.waitForProcessGroupExit(pgid, waitDeadline)
540
549
 
541
- if (!exited && retirementDeadline !== undefined && gracefulDeadline === undefined) {
550
+ if (!exited && retirementBound) {
542
551
  this.retirementTimedOut = true
543
552
  this.markGroupOutlivedStop(pgid)
544
553
  this.logger("process retirement wait timed out; leaving process alive", {id: this.id, pid: pgid, timeoutMs: options.retirementTimeoutMs})
@@ -560,6 +569,49 @@ export default class ManagedProcess extends EventEmitter {
560
569
  this.state = "stopped"
561
570
  }
562
571
 
572
+ /**
573
+ * Delivers the SIGKILL that retirement proved it must not wait for.
574
+ *
575
+ * Retirement deliberately leaves a non-blocking process alive: STOE-1077 measured what a
576
+ * stop-timeout kill costs when the process still holds live lanes (67 builds, 27.3
577
+ * build-hours). That is right while the process can still drain and wrong once it cannot — a
578
+ * wedged generation keeps its release open, keeps its port reservation, and makes every
579
+ * later deploy wait on it until an operator signals the exact PIDs by hand. This is the
580
+ * acting half of that decision, and it re-checks the two invariants this class owns before
581
+ * sending anything: the process must already be recorded as past its retirement budget, and
582
+ * its group must still have a member that can be signalled.
583
+ * @param {{reason: string}} options - Named evidence behind the escalation.
584
+ * @returns {Promise<boolean>} True when a live process group was signalled.
585
+ */
586
+ async escalateRetirement({reason}) {
587
+ const pgid = this.pid ?? this.groupOutlivedStop
588
+
589
+ if (pgid === undefined) return false
590
+
591
+ if (!this.retirementTimedOut) {
592
+ this.logger("retirement escalation refused; process is not past its retirement budget", {id: this.id, pid: pgid})
593
+
594
+ return false
595
+ }
596
+
597
+ if (!this.processGroupExists(pgid)) {
598
+ this.groupOutlivedStop = undefined
599
+ this.heldProcessGroupCount = undefined
600
+
601
+ return false
602
+ }
603
+
604
+ this.logger("process retirement escalated to SIGKILL", {id: this.id, pid: pgid, reason})
605
+
606
+ const killDeadline = Date.now() + ESCALATION_KILL_CONFIRM_MS
607
+
608
+ await this.signalProcessGroup("SIGKILL", pgid, killDeadline)
609
+
610
+ if (!await this.waitForProcessGroupExit(pgid, killDeadline)) this.markGroupOutlivedStop(pgid)
611
+
612
+ return true
613
+ }
614
+
563
615
  /** @returns {Promise<void>} Stops restarts and waits until the process has stopped accepting new work. */
564
616
  async quiesce() {
565
617
  if (this.quiescePromise) return await this.quiescePromise
@@ -702,6 +754,19 @@ export default class ManagedProcess extends EventEmitter {
702
754
  return this.stopTimeoutMs
703
755
  }
704
756
 
757
+ /**
758
+ * Reports whether this process's graceful window has no upper bound.
759
+ *
760
+ * `"indefinite"` is the consumer's statement that the supervisor must never kill this process;
761
+ * it is not a statement that a caller may wait for it forever. A stop that has no retirement
762
+ * budget for such a process can only hang, and when that stop runs inside an owner mutation it
763
+ * locks out every later mutation (STOE-2015).
764
+ * @returns {boolean} True when no deadline can ever end the wait.
765
+ */
766
+ hasUnboundedGracefulWindow() {
767
+ return this.stopTimeoutMs === "indefinite"
768
+ }
769
+
705
770
  /**
706
771
  * Runs a lifecycle hook command, bounded by a timeout so a hung hook can never block stop().
707
772
  * Failures are logged and returned so generation retirement can surface a failed quiet hook;
@@ -2,7 +2,7 @@
2
2
 
3
3
  import {spawn} from "node:child_process"
4
4
  import {EventEmitter} from "node:events"
5
- import {classifyDrainProgress} from "./drain-policy.js"
5
+ import {classifyDrainProgress, decideWedgedEscalation} from "./drain-policy.js"
6
6
  import ManagedProcess from "./managed-process.js"
7
7
  import {findAvailablePort} from "./port-allocator.js"
8
8
  import {processStartOrder} from "./process-order.js"
@@ -863,6 +863,15 @@ export default class ReleaseGroup extends EventEmitter {
863
863
  }))
864
864
  const handoffServices = entries.filter(([id]) => this.handoffServiceIds.has(id)).map(([, processInstance]) => processInstance)
865
865
  const connectionDependent = entries.filter(([id]) => !this.nonBlockingDrainIds.has(id) && !this.handoffServiceIds.has(id)).map(([, processInstance]) => processInstance)
866
+ // Bracket the retirement wait with a pair of observations: this one, and the one taken when
867
+ // every stop has returned. The gap between them is the wait the transition already performs
868
+ // (nonBlockingDrainTimeoutMs at minimum), so CPU flatness can be proven from the transition
869
+ // itself rather than from a sampler running alongside it.
870
+ const retirementBaselineSample = config.releaseRetention.escalateWedgedGenerations && nonBlockingStops.length > 0
871
+ ? this.takeLivenessSample(Date.now())
872
+ : undefined
873
+
874
+ if (retirementBaselineSample) this.drainNewestSample = retirementBaselineSample
866
875
 
867
876
  if (this.connectionCount > 0) {
868
877
  await new Promise((resolve) => {
@@ -899,11 +908,21 @@ export default class ReleaseGroup extends EventEmitter {
899
908
  }
900
909
  })
901
910
  await Promise.allSettled(handoffServices.map((processInstance) => processInstance.stop()))
911
+
912
+ // Last act of the retirement transition, before the release is recorded stopped: whatever
913
+ // is still alive here is the residue, and the bracket pair above can now say whether it is
914
+ // wedged. A generation that stops with a live process group is the state that made
915
+ // operators signal PIDs by hand (STOE-1687), so the transition decides it or reports it —
916
+ // it does not end silently with the process still holding the release.
917
+ if (retirementBaselineSample) await this.escalateWedgedGeneration({previousSample: retirementBaselineSample, reason: "release retirement"})
918
+
902
919
  this.state = "stopped"
903
920
  this.stoppedAt = new Date().toISOString()
904
921
  this.releasePortReservations()
905
922
  this.abandonLocalConnections()
906
- this.clearDrainLivenessSamples()
923
+ // The bracket pair is deliberately kept: a generation that stopped with a live process
924
+ // group stays observable, and the deploy path needs the older observation to prove the
925
+ // residue is still flat rather than starting a fresh window from zero.
907
926
  }
908
927
 
909
928
  /**
@@ -938,11 +957,39 @@ export default class ReleaseGroup extends EventEmitter {
938
957
  this.retirementError = undefined
939
958
  }
940
959
 
941
- /** @returns {Promise<void>} Stops all release-owned processes. */
942
- async stop() {
943
- const stopTasks = [...this.processes.values()].map((processInstance) => processInstance.stop())
960
+ /**
961
+ * Stops every process owned by the generation.
962
+ * @param {{retirementTimeoutMs?: number}} [options] - Retirement budget for the processes the
963
+ * supervisor must not kill: a non-blocking-drain process, and any process whose graceful window
964
+ * is `"indefinite"`. Without it those stops have no deadline at all, so an explicit
965
+ * `rollbridge stop` can wait forever — and because that command runs inside an owner mutation,
966
+ * the wait holds the mutation fence and every later deploy is refused (STOE-2015).
967
+ * @returns {Promise<void>} Resolves when stopped, or when retirement stops waiting.
968
+ */
969
+ async stop(options = {}) {
970
+ const {retirementTimeoutMs} = options
971
+ /** @type {{bounded: boolean, id: string, instance: ManagedProcess, promise: Promise<void>}[]} */
972
+ const stopTasks = [...this.processes.entries()].map(([id, processInstance]) => {
973
+ const bounded = retirementTimeoutMs !== undefined && (this.nonBlockingDrainIds.has(id) || processInstance.hasUnboundedGracefulWindow())
974
+
975
+ return {bounded, id, instance: processInstance, promise: processInstance.stop(bounded ? {retirementTimeoutMs} : {})}
976
+ })
977
+
978
+ await Promise.allSettled(stopTasks.map(({promise}) => promise))
979
+
980
+ // A stop that stopped waiting leaves the process running. Name it, so the operator sees the
981
+ // residue instead of a command that produced nothing.
982
+ for (const {bounded, id, instance} of stopTasks) {
983
+ if (!bounded || !instance.retirementTimedOut) continue
984
+
985
+ this.logger("release stop gave up waiting for a retiring process", {
986
+ pid: instance.status().pid ?? null,
987
+ processId: id,
988
+ releaseId: this.releaseId,
989
+ timeoutMs: retirementTimeoutMs
990
+ })
991
+ }
944
992
 
945
- await Promise.allSettled(stopTasks)
946
993
  this.state = "stopped"
947
994
  this.stoppedAt = new Date().toISOString()
948
995
  this.releasePortReservations()
@@ -969,32 +1016,109 @@ export default class ReleaseGroup extends EventEmitter {
969
1016
  drainStatus() {
970
1017
  if (this.state !== "draining") return undefined
971
1018
 
972
- const nowMs = Date.now()
1019
+ return this.observeRetirement({nowMs: Date.now()}).classification
1020
+ }
973
1021
 
974
- if (this.connectionCount > 0) {
975
- return classifyDrainProgress({
976
- connectionCount: this.connectionCount,
977
- currentSample: this.drainNewestSample ?? {atMs: nowMs, processes: []},
978
- minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
979
- previousSample: undefined
980
- })
1022
+ /**
1023
+ * Records the newest liveness observation of this generation and classifies its retirement.
1024
+ * @param {object} options - Observation options.
1025
+ * @param {number} options.nowMs - Read time.
1026
+ * @param {DrainLivenessSample | undefined} [options.previousSample] - Explicit older observation to classify against. The retirement transition brackets its own wait with one instead of reusing a stored pair.
1027
+ * @returns {{classification: ReleaseDrainStatus, currentSample: DrainLivenessSample}} The observation the decision acts on, and what the pair says.
1028
+ */
1029
+ observeRetirement({nowMs, previousSample}) {
1030
+ if (previousSample === undefined && this.connectionCount > 0) {
1031
+ const currentSample = this.drainNewestSample ?? {atMs: nowMs, processes: []}
1032
+
1033
+ return {
1034
+ classification: classifyDrainProgress({
1035
+ connectionCount: this.connectionCount,
1036
+ currentSample,
1037
+ minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
1038
+ previousSample: undefined
1039
+ }),
1040
+ currentSample
1041
+ }
981
1042
  }
982
1043
 
983
1044
  const newest = this.drainNewestSample
984
- const currentSample = newest && nowMs - newest.atMs < DRAIN_SAMPLE_WINDOW_MS ? newest : this.takeLivenessSample(nowMs)
985
- const previousSample = currentSample === newest ? this.drainPriorSample : newest
1045
+ const currentSample = previousSample !== undefined || newest === undefined || nowMs - newest.atMs >= DRAIN_SAMPLE_WINDOW_MS
1046
+ ? this.takeLivenessSample(nowMs)
1047
+ : newest
1048
+ const priorSample = previousSample ?? (currentSample === newest ? this.drainPriorSample : newest)
986
1049
 
987
1050
  if (currentSample !== newest) {
988
1051
  this.drainPriorSample = newest
989
1052
  this.drainNewestSample = currentSample
990
1053
  }
991
1054
 
992
- return classifyDrainProgress({
993
- connectionCount: this.connectionCount,
994
- currentSample,
995
- minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
996
- previousSample
997
- })
1055
+ return {
1056
+ classification: classifyDrainProgress({
1057
+ connectionCount: this.connectionCount,
1058
+ currentSample,
1059
+ minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
1060
+ previousSample: priorSample
1061
+ }),
1062
+ currentSample
1063
+ }
1064
+ }
1065
+
1066
+ /**
1067
+ * Applies the wedge escalation to this generation: terminates the processes a proven wedge
1068
+ * proves are holding nothing, and names the ones it refused to terminate.
1069
+ *
1070
+ * Both halves are required. The card this path was written for (STOE-1687) is about a
1071
+ * wedged worker holding a release open for four hours while deploys stack behind it; the
1072
+ * boundary on the same card (STOE-1077) is that no process still holding live work may ever
1073
+ * be SIGKILLed. So a wedge that is not kill-safe is not silently skipped — it is reported
1074
+ * with the pids and process groups it is holding, which is the information an operator needs
1075
+ * in place of guessing.
1076
+ * @param {object} options - Escalation context.
1077
+ * @param {DrainLivenessSample | undefined} [options.previousSample] - Older observation bracketing a wait this caller already performed.
1078
+ * @param {string} options.reason - Named context recorded with the decision.
1079
+ * @returns {Promise<string[]>} Instance ids of the processes the SIGKILL was delivered to.
1080
+ */
1081
+ async escalateWedgedGeneration({previousSample, reason}) {
1082
+ const {classification, currentSample} = this.observeRetirement({nowMs: Date.now(), previousSample})
1083
+ const decision = decideWedgedEscalation({classification, currentSample, minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS})
1084
+ const liveObservations = currentSample.processes.filter((process) => process.pid !== undefined || process.heldProcessGroupPgid !== undefined)
1085
+
1086
+ if (decision.decision === "report") {
1087
+ if (classification.status === "wedged") {
1088
+ this.logger("release wedged generation left alive", {
1089
+ escalation: "refused",
1090
+ heldProcessGroupPgids: liveObservations.map((process) => process.heldProcessGroupPgid ?? null),
1091
+ pids: liveObservations.map((process) => process.pid ?? null),
1092
+ processIds: liveObservations.map((process) => process.id),
1093
+ reason: decision.reason,
1094
+ releaseId: this.releaseId,
1095
+ trigger: reason
1096
+ })
1097
+ }
1098
+
1099
+ return []
1100
+ }
1101
+
1102
+ const escalated = []
1103
+
1104
+ for (const target of decision.targets) {
1105
+ const processInstance = this.processes.get(target.processId)
1106
+
1107
+ if (!processInstance) continue
1108
+ if (await processInstance.escalateRetirement({reason: `${reason}: ${decision.reason}`})) escalated.push(target.processId)
1109
+ }
1110
+
1111
+ if (escalated.length > 0) {
1112
+ this.logger("release wedged generation escalated", {
1113
+ escalation: "SIGKILL",
1114
+ processIds: escalated,
1115
+ reason: decision.reason,
1116
+ releaseId: this.releaseId,
1117
+ trigger: reason
1118
+ })
1119
+ }
1120
+
1121
+ return escalated
998
1122
  }
999
1123
 
1000
1124
  /**
@@ -705,12 +705,12 @@ test("validateConfig defaults releaseRetention, accepts overrides, and rejects b
705
705
  const defaulted = validateRetention(undefined)
706
706
 
707
707
  expect(defaulted.issues).toEqual([])
708
- expect(defaulted.config.releaseRetention).toEqual({keep: 10, maxAgeMs: 0, maxConcurrentDrainingReleaseGroups: 12})
708
+ expect(defaulted.config.releaseRetention).toEqual({escalateWedgedGenerations: false, keep: 10, maxAgeMs: 0, maxConcurrentDrainingReleaseGroups: 12})
709
709
 
710
- const custom = validateRetention({keep: 3, maxAgeMs: 60000, maxConcurrentDrainingReleaseGroups: 4})
710
+ const custom = validateRetention({escalateWedgedGenerations: true, keep: 3, maxAgeMs: 60000, maxConcurrentDrainingReleaseGroups: 4})
711
711
 
712
712
  expect(custom.issues).toEqual([])
713
- expect(custom.config.releaseRetention).toEqual({keep: 3, maxAgeMs: 60000, maxConcurrentDrainingReleaseGroups: 4})
713
+ expect(custom.config.releaseRetention).toEqual({escalateWedgedGenerations: true, keep: 3, maxAgeMs: 60000, maxConcurrentDrainingReleaseGroups: 4})
714
714
 
715
715
  // 0 is a valid bound: it disables the pre-activation draining-generation check.
716
716
  const unbounded = validateRetention({maxConcurrentDrainingReleaseGroups: 0})
@@ -718,8 +718,9 @@ test("validateConfig defaults releaseRetention, accepts overrides, and rejects b
718
718
  expect(unbounded.issues).toEqual([])
719
719
  expect(unbounded.config.releaseRetention.maxConcurrentDrainingReleaseGroups).toBe(0)
720
720
 
721
- const invalid = validateRetention({keep: -1, maxAgeMs: -5, maxConcurrentDrainingReleaseGroups: -2})
721
+ const invalid = validateRetention({escalateWedgedGenerations: "yes", keep: -1, maxAgeMs: -5, maxConcurrentDrainingReleaseGroups: -2})
722
722
 
723
+ expect(invalid.issues.some((issue) => issue.message === "releaseRetention.escalateWedgedGenerations must be a boolean")).toBeTruthy()
723
724
  expect(invalid.issues.some((issue) => issue.message === "releaseRetention.keep must be a non-negative integer")).toBeTruthy()
724
725
  expect(invalid.issues.some((issue) => issue.message === "releaseRetention.maxAgeMs must be a non-negative number")).toBeTruthy()
725
726
  expect(invalid.issues.some((issue) => issue.message === "releaseRetention.maxConcurrentDrainingReleaseGroups must be a non-negative integer")).toBeTruthy()
@@ -1,7 +1,7 @@
1
1
  // @ts-check
2
2
 
3
3
  import {describe, expect, test} from "@velocious/testing"
4
- import {admitGenerationRelease, classifyDrainProgress} from "../src/drain-policy.js"
4
+ import {admitGenerationRelease, classifyDrainProgress, decideWedgedEscalation} from "../src/drain-policy.js"
5
5
 
6
6
  describe("drain-policy", () => {
7
7
 
@@ -47,6 +47,23 @@ function classify(options = {}) {
47
47
  })
48
48
  }
49
49
 
50
+ /**
51
+ * @param {object} [options] - Decision inputs.
52
+ * @param {number} [options.connectionCount] - Live connections owned by the generation.
53
+ * @param {import("../src/drain-policy.js").DrainLivenessSample} [options.current] - Newest observation.
54
+ * @param {import("../src/drain-policy.js").DrainLivenessSample | undefined} [options.previous] - Older observation.
55
+ * @returns {import("../src/drain-policy.js").EscalationDecision} Escalation decision for the same pair the classifier saw.
56
+ */
57
+ function escalate(options = {}) {
58
+ const currentSample = options.current ?? sample()
59
+
60
+ return decideWedgedEscalation({
61
+ classification: classify({connectionCount: options.connectionCount, current: currentSample, previous: options.previous}),
62
+ currentSample,
63
+ minSampleWindowMs: WINDOW_MS
64
+ })
65
+ }
66
+
50
67
  test("a generation that still owns connections is draining, never wedged", () => {
51
68
  const result = classify({connectionCount: 3, current: sample({liveChildCount: 0})})
52
69
 
@@ -151,6 +168,90 @@ test("a held process group that still consumes CPU across the window is a progre
151
168
  expect(result.reason).toBe("CPU advanced 4000ms across 25000ms")
152
169
  })
153
170
 
171
+ test("a proven wedge with no runner child left is escalated, naming the process group it terminates", () => {
172
+ const previous = sample({atMs: 1000000, cpuMs: 5000, liveChildCount: 3})
173
+ const current = sample({atMs: 1025000, cpuMs: 5000, liveChildCount: 0})
174
+ const decision = escalate({current, previous})
175
+
176
+ expect(decision.decision).toBe("escalate")
177
+ expect(decision.killSafe).toBe(true)
178
+ expect(decision.targets).toEqual([{pgid: 4242, processId: "worker"}])
179
+ expect(decision.reason).toContain("CPU flat across 25000ms")
180
+ })
181
+
182
+ test("the escalation targets the group that outlived its tracked process", () => {
183
+ const previous = sample({atMs: 1000000, cpuMs: 5000, heldProcessGroupPgid: 4242, liveChildCount: 2, pid: undefined})
184
+ const current = sample({atMs: 1025000, cpuMs: 5000, heldProcessGroupPgid: 4242, liveChildCount: 0, pid: undefined})
185
+ const decision = escalate({current, previous})
186
+
187
+ expect(decision.decision).toBe("escalate")
188
+ expect(decision.targets).toEqual([{pgid: 4242, processId: "worker"}])
189
+ })
190
+
191
+ test("a wedge that still holds a live runner child is never escalated", () => {
192
+ const previous = sample({atMs: 1000000, cpuMs: 5000})
193
+ const current = sample({atMs: 1025000, cpuMs: 5000, liveChildCount: 3})
194
+
195
+ expect(classify({current, previous}).status).toBe("wedged")
196
+
197
+ const decision = escalate({current, previous})
198
+
199
+ expect(decision.decision).toBe("report")
200
+ expect(decision.killSafe).toBe(false)
201
+ expect(decision.targets).toEqual([])
202
+ expect(decision.reason).toBe("no escalation: 3 runner children are still live in the generation; SIGKILL would abort in-flight work")
203
+ })
204
+
205
+ test("a live process whose CPU is still moving is never escalated", () => {
206
+ const previous = sample({atMs: 1000000, cpuMs: 5000})
207
+ const current = sample({atMs: 1025000, cpuMs: 9000, liveChildCount: 0})
208
+
209
+ expect(classify({current, previous}).status).toBe("wedged")
210
+
211
+ const decision = escalate({current, previous})
212
+
213
+ expect(decision.decision).toBe("report")
214
+ expect(decision.reason).toBe("no escalation: CPU flatness is not proven (4000ms of CPU moved across 25000ms)")
215
+ })
216
+
217
+ test("a process that has not expired its retirement budget is never escalated", () => {
218
+ const previous = sample({atMs: 1000000, cpuMs: 5000, liveChildCount: 0, retirementTimedOut: false})
219
+ const current = sample({atMs: 1025000, cpuMs: 5000, liveChildCount: 0, retirementTimedOut: false})
220
+ const decision = escalate({current, previous})
221
+
222
+ expect(decision.decision).toBe("report")
223
+ expect(decision.reason).toBe("no escalation: drain is progressing (process worker is still inside its retirement budget)")
224
+ })
225
+
226
+ test("a generation that still owns a live connection is never escalated", () => {
227
+ const decision = escalate({connectionCount: 2, current: sample({liveChildCount: 0})})
228
+
229
+ expect(decision.decision).toBe("report")
230
+ expect(decision.reason).toBe("no escalation: drain is progressing (2 live connections still owned by the generation)")
231
+ })
232
+
233
+ test("a wedge with no measurable CPU is never escalated", () => {
234
+ const previous = sample({atMs: 1000000, cpuMs: undefined, liveChildCount: 0})
235
+ const current = sample({atMs: 1025000, cpuMs: undefined, liveChildCount: 0})
236
+
237
+ expect(classify({current, previous}).status).toBe("wedged")
238
+
239
+ const decision = escalate({current, previous})
240
+
241
+ expect(decision.decision).toBe("report")
242
+ expect(decision.reason).toBe("no escalation: CPU flatness is not proven (process CPU is not measurable on this host)")
243
+ })
244
+
245
+ test("a wedge seen in one observation is not escalated until a full window has passed", () => {
246
+ const current = sample({liveChildCount: 0})
247
+ const shortWindow = escalate({current: sample({atMs: 1001000, cpuMs: 5000, liveChildCount: 0}), previous: sample({atMs: 1000000, cpuMs: 5000, liveChildCount: 0})})
248
+
249
+ expect(classify({current}).status).toBe("wedged")
250
+ expect(escalate({current}).decision).toBe("report")
251
+ expect(escalate({current}).reason).toBe("no escalation: liveness window 0ms is below the 20000ms minimum")
252
+ expect(shortWindow.reason).toBe("no escalation: liveness window 1000ms is below the 20000ms minimum")
253
+ })
254
+
154
255
  test("admission allows a handoff that stays within the bound and counts the incumbent", () => {
155
256
  const admission = admitGenerationRelease({
156
257
  candidateReleaseId: "v3",