rollbridge 0.1.65 → 0.1.67
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +27 -0
- package/changelog.d/20261115-wedged-generation-escalation.md +44 -0
- package/changelog.d/20261116-bounded-explicit-stop-retirement.md +30 -0
- package/docs/cli.md +16 -1
- package/docs/config.md +47 -1
- package/docs/troubleshooting.md +24 -0
- package/docs/workers.md +10 -0
- package/package.json +1 -1
- package/src/config.js +2 -1
- package/src/daemon.js +124 -24
- package/src/drain-policy.js +79 -0
- package/src/managed-process.js +71 -6
- package/src/release-group.js +146 -22
- package/test/config-validation.test.js +5 -4
- package/test/drain-policy.test.js +102 -1
- package/test/managed-process.test.js +205 -1
- package/test/release-group.test.js +196 -0
- package/test/rollbridge.test.js +173 -0
package/src/managed-process.js
CHANGED
|
@@ -9,6 +9,10 @@ const MAX_BUFFERED_OUTPUT_CHARACTERS = 64 * 1024
|
|
|
9
9
|
// Reading a process group means scanning procfs, and status() is read continuously. A held
|
|
10
10
|
// group is re-verified on this interval instead of on every read.
|
|
11
11
|
const HELD_GROUP_RECHECK_MS = 5000
|
|
12
|
+
// How long an escalated SIGKILL is given to clear the group before the group is recorded as
|
|
13
|
+
// outliving it again. Bounded so the escalation itself can never become the unbounded wait it
|
|
14
|
+
// replaced.
|
|
15
|
+
const ESCALATION_KILL_CONFIRM_MS = 5000
|
|
12
16
|
|
|
13
17
|
/**
|
|
14
18
|
* @typedef {import("./json.js").JsonValue} JsonValue
|
|
@@ -525,20 +529,25 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
525
529
|
if (this.processGroupExists(pgid)) {
|
|
526
530
|
const timeoutMs = options.timeoutMs ?? this.stopTimeoutMs
|
|
527
531
|
const retirementDeadline = options.retirementTimeoutMs === undefined ? undefined : Date.now() + options.retirementTimeoutMs
|
|
528
|
-
|
|
532
|
+
const gracefulDeadline = timeoutMs === "indefinite" ? undefined : Date.now() + timeoutMs
|
|
533
|
+
// A retirement budget is the caller's decision that this process must not be waited for and
|
|
534
|
+
// must not be killed, so it caps the graceful window rather than only standing in for a
|
|
535
|
+
// missing one. Read the other way round, a finite `gracefulStopMs` outvotes the retirement
|
|
536
|
+
// bound and the wait runs for the whole window: TensorBuzz production moved its worker from
|
|
537
|
+
// `"indefinite"` to 4,5 h and the stop would still have held the control plane for 4,5 h.
|
|
538
|
+
const retirementBound = retirementDeadline !== undefined && (gracefulDeadline === undefined || retirementDeadline < gracefulDeadline)
|
|
539
|
+
const waitDeadline = retirementBound ? retirementDeadline : gracefulDeadline
|
|
529
540
|
|
|
530
541
|
if (stopCommand) {
|
|
531
542
|
await this.runHook(stopCommand, hookTimeoutMs, "stop command", pgid)
|
|
532
|
-
gracefulDeadline = timeoutMs === "indefinite" ? undefined : Date.now() + timeoutMs
|
|
533
543
|
} else {
|
|
534
544
|
this.intentionalStopSignal = /** @type {ProcessExitSignal} */ (this.stopSignal)
|
|
535
|
-
|
|
536
|
-
await this.signalProcessGroup(this.stopSignal, pgid, gracefulDeadline ?? retirementDeadline)
|
|
545
|
+
await this.signalProcessGroup(this.stopSignal, pgid, waitDeadline)
|
|
537
546
|
}
|
|
538
547
|
|
|
539
|
-
const exited = await this.waitForProcessGroupExit(pgid,
|
|
548
|
+
const exited = await this.waitForProcessGroupExit(pgid, waitDeadline)
|
|
540
549
|
|
|
541
|
-
if (!exited &&
|
|
550
|
+
if (!exited && retirementBound) {
|
|
542
551
|
this.retirementTimedOut = true
|
|
543
552
|
this.markGroupOutlivedStop(pgid)
|
|
544
553
|
this.logger("process retirement wait timed out; leaving process alive", {id: this.id, pid: pgid, timeoutMs: options.retirementTimeoutMs})
|
|
@@ -560,6 +569,49 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
560
569
|
this.state = "stopped"
|
|
561
570
|
}
|
|
562
571
|
|
|
572
|
+
/**
|
|
573
|
+
* Delivers the SIGKILL that retirement proved it must not wait for.
|
|
574
|
+
*
|
|
575
|
+
* Retirement deliberately leaves a non-blocking process alive: STOE-1077 measured what a
|
|
576
|
+
* stop-timeout kill costs when the process still holds live lanes (67 builds, 27.3
|
|
577
|
+
* build-hours). That is right while the process can still drain and wrong once it cannot — a
|
|
578
|
+
* wedged generation keeps its release open, keeps its port reservation, and makes every
|
|
579
|
+
* later deploy wait on it until an operator signals the exact PIDs by hand. This is the
|
|
580
|
+
* acting half of that decision, and it re-checks the two invariants this class owns before
|
|
581
|
+
* sending anything: the process must already be recorded as past its retirement budget, and
|
|
582
|
+
* its group must still have a member that can be signalled.
|
|
583
|
+
* @param {{reason: string}} options - Named evidence behind the escalation.
|
|
584
|
+
* @returns {Promise<boolean>} True when a live process group was signalled.
|
|
585
|
+
*/
|
|
586
|
+
async escalateRetirement({reason}) {
|
|
587
|
+
const pgid = this.pid ?? this.groupOutlivedStop
|
|
588
|
+
|
|
589
|
+
if (pgid === undefined) return false
|
|
590
|
+
|
|
591
|
+
if (!this.retirementTimedOut) {
|
|
592
|
+
this.logger("retirement escalation refused; process is not past its retirement budget", {id: this.id, pid: pgid})
|
|
593
|
+
|
|
594
|
+
return false
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
if (!this.processGroupExists(pgid)) {
|
|
598
|
+
this.groupOutlivedStop = undefined
|
|
599
|
+
this.heldProcessGroupCount = undefined
|
|
600
|
+
|
|
601
|
+
return false
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
this.logger("process retirement escalated to SIGKILL", {id: this.id, pid: pgid, reason})
|
|
605
|
+
|
|
606
|
+
const killDeadline = Date.now() + ESCALATION_KILL_CONFIRM_MS
|
|
607
|
+
|
|
608
|
+
await this.signalProcessGroup("SIGKILL", pgid, killDeadline)
|
|
609
|
+
|
|
610
|
+
if (!await this.waitForProcessGroupExit(pgid, killDeadline)) this.markGroupOutlivedStop(pgid)
|
|
611
|
+
|
|
612
|
+
return true
|
|
613
|
+
}
|
|
614
|
+
|
|
563
615
|
/** @returns {Promise<void>} Stops restarts and waits until the process has stopped accepting new work. */
|
|
564
616
|
async quiesce() {
|
|
565
617
|
if (this.quiescePromise) return await this.quiescePromise
|
|
@@ -702,6 +754,19 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
702
754
|
return this.stopTimeoutMs
|
|
703
755
|
}
|
|
704
756
|
|
|
757
|
+
/**
|
|
758
|
+
* Reports whether this process's graceful window has no upper bound.
|
|
759
|
+
*
|
|
760
|
+
* `"indefinite"` is the consumer's statement that the supervisor must never kill this process;
|
|
761
|
+
* it is not a statement that a caller may wait for it forever. A stop that has no retirement
|
|
762
|
+
* budget for such a process can only hang, and when that stop runs inside an owner mutation it
|
|
763
|
+
* locks out every later mutation (STOE-2015).
|
|
764
|
+
* @returns {boolean} True when no deadline can ever end the wait.
|
|
765
|
+
*/
|
|
766
|
+
hasUnboundedGracefulWindow() {
|
|
767
|
+
return this.stopTimeoutMs === "indefinite"
|
|
768
|
+
}
|
|
769
|
+
|
|
705
770
|
/**
|
|
706
771
|
* Runs a lifecycle hook command, bounded by a timeout so a hung hook can never block stop().
|
|
707
772
|
* Failures are logged and returned so generation retirement can surface a failed quiet hook;
|
package/src/release-group.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import {spawn} from "node:child_process"
|
|
4
4
|
import {EventEmitter} from "node:events"
|
|
5
|
-
import {classifyDrainProgress} from "./drain-policy.js"
|
|
5
|
+
import {classifyDrainProgress, decideWedgedEscalation} from "./drain-policy.js"
|
|
6
6
|
import ManagedProcess from "./managed-process.js"
|
|
7
7
|
import {findAvailablePort} from "./port-allocator.js"
|
|
8
8
|
import {processStartOrder} from "./process-order.js"
|
|
@@ -863,6 +863,15 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
863
863
|
}))
|
|
864
864
|
const handoffServices = entries.filter(([id]) => this.handoffServiceIds.has(id)).map(([, processInstance]) => processInstance)
|
|
865
865
|
const connectionDependent = entries.filter(([id]) => !this.nonBlockingDrainIds.has(id) && !this.handoffServiceIds.has(id)).map(([, processInstance]) => processInstance)
|
|
866
|
+
// Bracket the retirement wait with a pair of observations: this one, and the one taken when
|
|
867
|
+
// every stop has returned. The gap between them is the wait the transition already performs
|
|
868
|
+
// (nonBlockingDrainTimeoutMs at minimum), so CPU flatness can be proven from the transition
|
|
869
|
+
// itself rather than from a sampler running alongside it.
|
|
870
|
+
const retirementBaselineSample = config.releaseRetention.escalateWedgedGenerations && nonBlockingStops.length > 0
|
|
871
|
+
? this.takeLivenessSample(Date.now())
|
|
872
|
+
: undefined
|
|
873
|
+
|
|
874
|
+
if (retirementBaselineSample) this.drainNewestSample = retirementBaselineSample
|
|
866
875
|
|
|
867
876
|
if (this.connectionCount > 0) {
|
|
868
877
|
await new Promise((resolve) => {
|
|
@@ -899,11 +908,21 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
899
908
|
}
|
|
900
909
|
})
|
|
901
910
|
await Promise.allSettled(handoffServices.map((processInstance) => processInstance.stop()))
|
|
911
|
+
|
|
912
|
+
// Last act of the retirement transition, before the release is recorded stopped: whatever
|
|
913
|
+
// is still alive here is the residue, and the bracket pair above can now say whether it is
|
|
914
|
+
// wedged. A generation that stops with a live process group is the state that made
|
|
915
|
+
// operators signal PIDs by hand (STOE-1687), so the transition decides it or reports it —
|
|
916
|
+
// it does not end silently with the process still holding the release.
|
|
917
|
+
if (retirementBaselineSample) await this.escalateWedgedGeneration({previousSample: retirementBaselineSample, reason: "release retirement"})
|
|
918
|
+
|
|
902
919
|
this.state = "stopped"
|
|
903
920
|
this.stoppedAt = new Date().toISOString()
|
|
904
921
|
this.releasePortReservations()
|
|
905
922
|
this.abandonLocalConnections()
|
|
906
|
-
|
|
923
|
+
// The bracket pair is deliberately kept: a generation that stopped with a live process
|
|
924
|
+
// group stays observable, and the deploy path needs the older observation to prove the
|
|
925
|
+
// residue is still flat rather than starting a fresh window from zero.
|
|
907
926
|
}
|
|
908
927
|
|
|
909
928
|
/**
|
|
@@ -938,11 +957,39 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
938
957
|
this.retirementError = undefined
|
|
939
958
|
}
|
|
940
959
|
|
|
941
|
-
/**
|
|
942
|
-
|
|
943
|
-
|
|
960
|
+
/**
|
|
961
|
+
* Stops every process owned by the generation.
|
|
962
|
+
* @param {{retirementTimeoutMs?: number}} [options] - Retirement budget for the processes the
|
|
963
|
+
* supervisor must not kill: a non-blocking-drain process, and any process whose graceful window
|
|
964
|
+
* is `"indefinite"`. Without it those stops have no deadline at all, so an explicit
|
|
965
|
+
* `rollbridge stop` can wait forever — and because that command runs inside an owner mutation,
|
|
966
|
+
* the wait holds the mutation fence and every later deploy is refused (STOE-2015).
|
|
967
|
+
* @returns {Promise<void>} Resolves when stopped, or when retirement stops waiting.
|
|
968
|
+
*/
|
|
969
|
+
async stop(options = {}) {
|
|
970
|
+
const {retirementTimeoutMs} = options
|
|
971
|
+
/** @type {{bounded: boolean, id: string, instance: ManagedProcess, promise: Promise<void>}[]} */
|
|
972
|
+
const stopTasks = [...this.processes.entries()].map(([id, processInstance]) => {
|
|
973
|
+
const bounded = retirementTimeoutMs !== undefined && (this.nonBlockingDrainIds.has(id) || processInstance.hasUnboundedGracefulWindow())
|
|
974
|
+
|
|
975
|
+
return {bounded, id, instance: processInstance, promise: processInstance.stop(bounded ? {retirementTimeoutMs} : {})}
|
|
976
|
+
})
|
|
977
|
+
|
|
978
|
+
await Promise.allSettled(stopTasks.map(({promise}) => promise))
|
|
979
|
+
|
|
980
|
+
// A stop that stopped waiting leaves the process running. Name it, so the operator sees the
|
|
981
|
+
// residue instead of a command that produced nothing.
|
|
982
|
+
for (const {bounded, id, instance} of stopTasks) {
|
|
983
|
+
if (!bounded || !instance.retirementTimedOut) continue
|
|
984
|
+
|
|
985
|
+
this.logger("release stop gave up waiting for a retiring process", {
|
|
986
|
+
pid: instance.status().pid ?? null,
|
|
987
|
+
processId: id,
|
|
988
|
+
releaseId: this.releaseId,
|
|
989
|
+
timeoutMs: retirementTimeoutMs
|
|
990
|
+
})
|
|
991
|
+
}
|
|
944
992
|
|
|
945
|
-
await Promise.allSettled(stopTasks)
|
|
946
993
|
this.state = "stopped"
|
|
947
994
|
this.stoppedAt = new Date().toISOString()
|
|
948
995
|
this.releasePortReservations()
|
|
@@ -969,32 +1016,109 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
969
1016
|
drainStatus() {
|
|
970
1017
|
if (this.state !== "draining") return undefined
|
|
971
1018
|
|
|
972
|
-
|
|
1019
|
+
return this.observeRetirement({nowMs: Date.now()}).classification
|
|
1020
|
+
}
|
|
973
1021
|
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
1022
|
+
/**
|
|
1023
|
+
* Records the newest liveness observation of this generation and classifies its retirement.
|
|
1024
|
+
* @param {object} options - Observation options.
|
|
1025
|
+
* @param {number} options.nowMs - Read time.
|
|
1026
|
+
* @param {DrainLivenessSample | undefined} [options.previousSample] - Explicit older observation to classify against. The retirement transition brackets its own wait with one instead of reusing a stored pair.
|
|
1027
|
+
* @returns {{classification: ReleaseDrainStatus, currentSample: DrainLivenessSample}} The observation the decision acts on, and what the pair says.
|
|
1028
|
+
*/
|
|
1029
|
+
observeRetirement({nowMs, previousSample}) {
|
|
1030
|
+
if (previousSample === undefined && this.connectionCount > 0) {
|
|
1031
|
+
const currentSample = this.drainNewestSample ?? {atMs: nowMs, processes: []}
|
|
1032
|
+
|
|
1033
|
+
return {
|
|
1034
|
+
classification: classifyDrainProgress({
|
|
1035
|
+
connectionCount: this.connectionCount,
|
|
1036
|
+
currentSample,
|
|
1037
|
+
minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
|
|
1038
|
+
previousSample: undefined
|
|
1039
|
+
}),
|
|
1040
|
+
currentSample
|
|
1041
|
+
}
|
|
981
1042
|
}
|
|
982
1043
|
|
|
983
1044
|
const newest = this.drainNewestSample
|
|
984
|
-
const currentSample = newest
|
|
985
|
-
|
|
1045
|
+
const currentSample = previousSample !== undefined || newest === undefined || nowMs - newest.atMs >= DRAIN_SAMPLE_WINDOW_MS
|
|
1046
|
+
? this.takeLivenessSample(nowMs)
|
|
1047
|
+
: newest
|
|
1048
|
+
const priorSample = previousSample ?? (currentSample === newest ? this.drainPriorSample : newest)
|
|
986
1049
|
|
|
987
1050
|
if (currentSample !== newest) {
|
|
988
1051
|
this.drainPriorSample = newest
|
|
989
1052
|
this.drainNewestSample = currentSample
|
|
990
1053
|
}
|
|
991
1054
|
|
|
992
|
-
return
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
1055
|
+
return {
|
|
1056
|
+
classification: classifyDrainProgress({
|
|
1057
|
+
connectionCount: this.connectionCount,
|
|
1058
|
+
currentSample,
|
|
1059
|
+
minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
|
|
1060
|
+
previousSample: priorSample
|
|
1061
|
+
}),
|
|
1062
|
+
currentSample
|
|
1063
|
+
}
|
|
1064
|
+
}
|
|
1065
|
+
|
|
1066
|
+
/**
|
|
1067
|
+
* Applies the wedge escalation to this generation: terminates the processes a proven wedge
|
|
1068
|
+
* proves are holding nothing, and names the ones it refused to terminate.
|
|
1069
|
+
*
|
|
1070
|
+
* Both halves are required. The card this path was written for (STOE-1687) is about a
|
|
1071
|
+
* wedged worker holding a release open for four hours while deploys stack behind it; the
|
|
1072
|
+
* boundary on the same card (STOE-1077) is that no process still holding live work may ever
|
|
1073
|
+
* be SIGKILLed. So a wedge that is not kill-safe is not silently skipped — it is reported
|
|
1074
|
+
* with the pids and process groups it is holding, which is the information an operator needs
|
|
1075
|
+
* in place of guessing.
|
|
1076
|
+
* @param {object} options - Escalation context.
|
|
1077
|
+
* @param {DrainLivenessSample | undefined} [options.previousSample] - Older observation bracketing a wait this caller already performed.
|
|
1078
|
+
* @param {string} options.reason - Named context recorded with the decision.
|
|
1079
|
+
* @returns {Promise<string[]>} Instance ids of the processes the SIGKILL was delivered to.
|
|
1080
|
+
*/
|
|
1081
|
+
async escalateWedgedGeneration({previousSample, reason}) {
|
|
1082
|
+
const {classification, currentSample} = this.observeRetirement({nowMs: Date.now(), previousSample})
|
|
1083
|
+
const decision = decideWedgedEscalation({classification, currentSample, minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS})
|
|
1084
|
+
const liveObservations = currentSample.processes.filter((process) => process.pid !== undefined || process.heldProcessGroupPgid !== undefined)
|
|
1085
|
+
|
|
1086
|
+
if (decision.decision === "report") {
|
|
1087
|
+
if (classification.status === "wedged") {
|
|
1088
|
+
this.logger("release wedged generation left alive", {
|
|
1089
|
+
escalation: "refused",
|
|
1090
|
+
heldProcessGroupPgids: liveObservations.map((process) => process.heldProcessGroupPgid ?? null),
|
|
1091
|
+
pids: liveObservations.map((process) => process.pid ?? null),
|
|
1092
|
+
processIds: liveObservations.map((process) => process.id),
|
|
1093
|
+
reason: decision.reason,
|
|
1094
|
+
releaseId: this.releaseId,
|
|
1095
|
+
trigger: reason
|
|
1096
|
+
})
|
|
1097
|
+
}
|
|
1098
|
+
|
|
1099
|
+
return []
|
|
1100
|
+
}
|
|
1101
|
+
|
|
1102
|
+
const escalated = []
|
|
1103
|
+
|
|
1104
|
+
for (const target of decision.targets) {
|
|
1105
|
+
const processInstance = this.processes.get(target.processId)
|
|
1106
|
+
|
|
1107
|
+
if (!processInstance) continue
|
|
1108
|
+
if (await processInstance.escalateRetirement({reason: `${reason}: ${decision.reason}`})) escalated.push(target.processId)
|
|
1109
|
+
}
|
|
1110
|
+
|
|
1111
|
+
if (escalated.length > 0) {
|
|
1112
|
+
this.logger("release wedged generation escalated", {
|
|
1113
|
+
escalation: "SIGKILL",
|
|
1114
|
+
processIds: escalated,
|
|
1115
|
+
reason: decision.reason,
|
|
1116
|
+
releaseId: this.releaseId,
|
|
1117
|
+
trigger: reason
|
|
1118
|
+
})
|
|
1119
|
+
}
|
|
1120
|
+
|
|
1121
|
+
return escalated
|
|
998
1122
|
}
|
|
999
1123
|
|
|
1000
1124
|
/**
|
|
@@ -705,12 +705,12 @@ test("validateConfig defaults releaseRetention, accepts overrides, and rejects b
|
|
|
705
705
|
const defaulted = validateRetention(undefined)
|
|
706
706
|
|
|
707
707
|
expect(defaulted.issues).toEqual([])
|
|
708
|
-
expect(defaulted.config.releaseRetention).toEqual({keep: 10, maxAgeMs: 0, maxConcurrentDrainingReleaseGroups: 12})
|
|
708
|
+
expect(defaulted.config.releaseRetention).toEqual({escalateWedgedGenerations: false, keep: 10, maxAgeMs: 0, maxConcurrentDrainingReleaseGroups: 12})
|
|
709
709
|
|
|
710
|
-
const custom = validateRetention({keep: 3, maxAgeMs: 60000, maxConcurrentDrainingReleaseGroups: 4})
|
|
710
|
+
const custom = validateRetention({escalateWedgedGenerations: true, keep: 3, maxAgeMs: 60000, maxConcurrentDrainingReleaseGroups: 4})
|
|
711
711
|
|
|
712
712
|
expect(custom.issues).toEqual([])
|
|
713
|
-
expect(custom.config.releaseRetention).toEqual({keep: 3, maxAgeMs: 60000, maxConcurrentDrainingReleaseGroups: 4})
|
|
713
|
+
expect(custom.config.releaseRetention).toEqual({escalateWedgedGenerations: true, keep: 3, maxAgeMs: 60000, maxConcurrentDrainingReleaseGroups: 4})
|
|
714
714
|
|
|
715
715
|
// 0 is a valid bound: it disables the pre-activation draining-generation check.
|
|
716
716
|
const unbounded = validateRetention({maxConcurrentDrainingReleaseGroups: 0})
|
|
@@ -718,8 +718,9 @@ test("validateConfig defaults releaseRetention, accepts overrides, and rejects b
|
|
|
718
718
|
expect(unbounded.issues).toEqual([])
|
|
719
719
|
expect(unbounded.config.releaseRetention.maxConcurrentDrainingReleaseGroups).toBe(0)
|
|
720
720
|
|
|
721
|
-
const invalid = validateRetention({keep: -1, maxAgeMs: -5, maxConcurrentDrainingReleaseGroups: -2})
|
|
721
|
+
const invalid = validateRetention({escalateWedgedGenerations: "yes", keep: -1, maxAgeMs: -5, maxConcurrentDrainingReleaseGroups: -2})
|
|
722
722
|
|
|
723
|
+
expect(invalid.issues.some((issue) => issue.message === "releaseRetention.escalateWedgedGenerations must be a boolean")).toBeTruthy()
|
|
723
724
|
expect(invalid.issues.some((issue) => issue.message === "releaseRetention.keep must be a non-negative integer")).toBeTruthy()
|
|
724
725
|
expect(invalid.issues.some((issue) => issue.message === "releaseRetention.maxAgeMs must be a non-negative number")).toBeTruthy()
|
|
725
726
|
expect(invalid.issues.some((issue) => issue.message === "releaseRetention.maxConcurrentDrainingReleaseGroups must be a non-negative integer")).toBeTruthy()
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
3
|
import {describe, expect, test} from "@velocious/testing"
|
|
4
|
-
import {admitGenerationRelease, classifyDrainProgress} from "../src/drain-policy.js"
|
|
4
|
+
import {admitGenerationRelease, classifyDrainProgress, decideWedgedEscalation} from "../src/drain-policy.js"
|
|
5
5
|
|
|
6
6
|
describe("drain-policy", () => {
|
|
7
7
|
|
|
@@ -47,6 +47,23 @@ function classify(options = {}) {
|
|
|
47
47
|
})
|
|
48
48
|
}
|
|
49
49
|
|
|
50
|
+
/**
|
|
51
|
+
* @param {object} [options] - Decision inputs.
|
|
52
|
+
* @param {number} [options.connectionCount] - Live connections owned by the generation.
|
|
53
|
+
* @param {import("../src/drain-policy.js").DrainLivenessSample} [options.current] - Newest observation.
|
|
54
|
+
* @param {import("../src/drain-policy.js").DrainLivenessSample | undefined} [options.previous] - Older observation.
|
|
55
|
+
* @returns {import("../src/drain-policy.js").EscalationDecision} Escalation decision for the same pair the classifier saw.
|
|
56
|
+
*/
|
|
57
|
+
function escalate(options = {}) {
|
|
58
|
+
const currentSample = options.current ?? sample()
|
|
59
|
+
|
|
60
|
+
return decideWedgedEscalation({
|
|
61
|
+
classification: classify({connectionCount: options.connectionCount, current: currentSample, previous: options.previous}),
|
|
62
|
+
currentSample,
|
|
63
|
+
minSampleWindowMs: WINDOW_MS
|
|
64
|
+
})
|
|
65
|
+
}
|
|
66
|
+
|
|
50
67
|
test("a generation that still owns connections is draining, never wedged", () => {
|
|
51
68
|
const result = classify({connectionCount: 3, current: sample({liveChildCount: 0})})
|
|
52
69
|
|
|
@@ -151,6 +168,90 @@ test("a held process group that still consumes CPU across the window is a progre
|
|
|
151
168
|
expect(result.reason).toBe("CPU advanced 4000ms across 25000ms")
|
|
152
169
|
})
|
|
153
170
|
|
|
171
|
+
test("a proven wedge with no runner child left is escalated, naming the process group it terminates", () => {
|
|
172
|
+
const previous = sample({atMs: 1000000, cpuMs: 5000, liveChildCount: 3})
|
|
173
|
+
const current = sample({atMs: 1025000, cpuMs: 5000, liveChildCount: 0})
|
|
174
|
+
const decision = escalate({current, previous})
|
|
175
|
+
|
|
176
|
+
expect(decision.decision).toBe("escalate")
|
|
177
|
+
expect(decision.killSafe).toBe(true)
|
|
178
|
+
expect(decision.targets).toEqual([{pgid: 4242, processId: "worker"}])
|
|
179
|
+
expect(decision.reason).toContain("CPU flat across 25000ms")
|
|
180
|
+
})
|
|
181
|
+
|
|
182
|
+
test("the escalation targets the group that outlived its tracked process", () => {
|
|
183
|
+
const previous = sample({atMs: 1000000, cpuMs: 5000, heldProcessGroupPgid: 4242, liveChildCount: 2, pid: undefined})
|
|
184
|
+
const current = sample({atMs: 1025000, cpuMs: 5000, heldProcessGroupPgid: 4242, liveChildCount: 0, pid: undefined})
|
|
185
|
+
const decision = escalate({current, previous})
|
|
186
|
+
|
|
187
|
+
expect(decision.decision).toBe("escalate")
|
|
188
|
+
expect(decision.targets).toEqual([{pgid: 4242, processId: "worker"}])
|
|
189
|
+
})
|
|
190
|
+
|
|
191
|
+
test("a wedge that still holds a live runner child is never escalated", () => {
|
|
192
|
+
const previous = sample({atMs: 1000000, cpuMs: 5000})
|
|
193
|
+
const current = sample({atMs: 1025000, cpuMs: 5000, liveChildCount: 3})
|
|
194
|
+
|
|
195
|
+
expect(classify({current, previous}).status).toBe("wedged")
|
|
196
|
+
|
|
197
|
+
const decision = escalate({current, previous})
|
|
198
|
+
|
|
199
|
+
expect(decision.decision).toBe("report")
|
|
200
|
+
expect(decision.killSafe).toBe(false)
|
|
201
|
+
expect(decision.targets).toEqual([])
|
|
202
|
+
expect(decision.reason).toBe("no escalation: 3 runner children are still live in the generation; SIGKILL would abort in-flight work")
|
|
203
|
+
})
|
|
204
|
+
|
|
205
|
+
test("a live process whose CPU is still moving is never escalated", () => {
|
|
206
|
+
const previous = sample({atMs: 1000000, cpuMs: 5000})
|
|
207
|
+
const current = sample({atMs: 1025000, cpuMs: 9000, liveChildCount: 0})
|
|
208
|
+
|
|
209
|
+
expect(classify({current, previous}).status).toBe("wedged")
|
|
210
|
+
|
|
211
|
+
const decision = escalate({current, previous})
|
|
212
|
+
|
|
213
|
+
expect(decision.decision).toBe("report")
|
|
214
|
+
expect(decision.reason).toBe("no escalation: CPU flatness is not proven (4000ms of CPU moved across 25000ms)")
|
|
215
|
+
})
|
|
216
|
+
|
|
217
|
+
test("a process that has not expired its retirement budget is never escalated", () => {
|
|
218
|
+
const previous = sample({atMs: 1000000, cpuMs: 5000, liveChildCount: 0, retirementTimedOut: false})
|
|
219
|
+
const current = sample({atMs: 1025000, cpuMs: 5000, liveChildCount: 0, retirementTimedOut: false})
|
|
220
|
+
const decision = escalate({current, previous})
|
|
221
|
+
|
|
222
|
+
expect(decision.decision).toBe("report")
|
|
223
|
+
expect(decision.reason).toBe("no escalation: drain is progressing (process worker is still inside its retirement budget)")
|
|
224
|
+
})
|
|
225
|
+
|
|
226
|
+
test("a generation that still owns a live connection is never escalated", () => {
|
|
227
|
+
const decision = escalate({connectionCount: 2, current: sample({liveChildCount: 0})})
|
|
228
|
+
|
|
229
|
+
expect(decision.decision).toBe("report")
|
|
230
|
+
expect(decision.reason).toBe("no escalation: drain is progressing (2 live connections still owned by the generation)")
|
|
231
|
+
})
|
|
232
|
+
|
|
233
|
+
test("a wedge with no measurable CPU is never escalated", () => {
|
|
234
|
+
const previous = sample({atMs: 1000000, cpuMs: undefined, liveChildCount: 0})
|
|
235
|
+
const current = sample({atMs: 1025000, cpuMs: undefined, liveChildCount: 0})
|
|
236
|
+
|
|
237
|
+
expect(classify({current, previous}).status).toBe("wedged")
|
|
238
|
+
|
|
239
|
+
const decision = escalate({current, previous})
|
|
240
|
+
|
|
241
|
+
expect(decision.decision).toBe("report")
|
|
242
|
+
expect(decision.reason).toBe("no escalation: CPU flatness is not proven (process CPU is not measurable on this host)")
|
|
243
|
+
})
|
|
244
|
+
|
|
245
|
+
test("a wedge seen in one observation is not escalated until a full window has passed", () => {
|
|
246
|
+
const current = sample({liveChildCount: 0})
|
|
247
|
+
const shortWindow = escalate({current: sample({atMs: 1001000, cpuMs: 5000, liveChildCount: 0}), previous: sample({atMs: 1000000, cpuMs: 5000, liveChildCount: 0})})
|
|
248
|
+
|
|
249
|
+
expect(classify({current}).status).toBe("wedged")
|
|
250
|
+
expect(escalate({current}).decision).toBe("report")
|
|
251
|
+
expect(escalate({current}).reason).toBe("no escalation: liveness window 0ms is below the 20000ms minimum")
|
|
252
|
+
expect(shortWindow.reason).toBe("no escalation: liveness window 1000ms is below the 20000ms minimum")
|
|
253
|
+
})
|
|
254
|
+
|
|
154
255
|
test("admission allows a handoff that stays within the bound and counts the incumbent", () => {
|
|
155
256
|
const admission = admitGenerationRelease({
|
|
156
257
|
candidateReleaseId: "v3",
|