rollbridge 0.1.66 → 0.1.68
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.d/20261115-wedged-generation-escalation.md +44 -0
- package/changelog.d/20261116-bounded-explicit-stop-retirement.md +30 -0
- package/changelog.d/20261116-config-json-canonical-shape.md +11 -0
- package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +29 -0
- package/docs/config.md +47 -1
- package/docs/troubleshooting.md +24 -0
- package/docs/workers.md +10 -0
- package/package.json +1 -1
- package/src/config.js +30 -3
- package/src/daemon.js +40 -3
- package/src/drain-policy.js +79 -0
- package/src/managed-process.js +71 -6
- package/src/release-group.js +146 -22
- package/test/compose-config.test.js +4 -2
- package/test/config-validation.test.js +26 -5
- package/test/drain-policy.test.js +102 -1
- package/test/managed-process.test.js +212 -1
- package/test/owner-replacement.test.js +56 -0
- package/test/release-group.test.js +196 -0
- package/test/rollbridge.test.js +125 -0
- package/test/support/process.js +3 -1
package/src/release-group.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import {spawn} from "node:child_process"
|
|
4
4
|
import {EventEmitter} from "node:events"
|
|
5
|
-
import {classifyDrainProgress} from "./drain-policy.js"
|
|
5
|
+
import {classifyDrainProgress, decideWedgedEscalation} from "./drain-policy.js"
|
|
6
6
|
import ManagedProcess from "./managed-process.js"
|
|
7
7
|
import {findAvailablePort} from "./port-allocator.js"
|
|
8
8
|
import {processStartOrder} from "./process-order.js"
|
|
@@ -863,6 +863,15 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
863
863
|
}))
|
|
864
864
|
const handoffServices = entries.filter(([id]) => this.handoffServiceIds.has(id)).map(([, processInstance]) => processInstance)
|
|
865
865
|
const connectionDependent = entries.filter(([id]) => !this.nonBlockingDrainIds.has(id) && !this.handoffServiceIds.has(id)).map(([, processInstance]) => processInstance)
|
|
866
|
+
// Bracket the retirement wait with a pair of observations: this one, and the one taken when
|
|
867
|
+
// every stop has returned. The gap between them is the wait the transition already performs
|
|
868
|
+
// (nonBlockingDrainTimeoutMs at minimum), so CPU flatness can be proven from the transition
|
|
869
|
+
// itself rather than from a sampler running alongside it.
|
|
870
|
+
const retirementBaselineSample = config.releaseRetention.escalateWedgedGenerations && nonBlockingStops.length > 0
|
|
871
|
+
? this.takeLivenessSample(Date.now())
|
|
872
|
+
: undefined
|
|
873
|
+
|
|
874
|
+
if (retirementBaselineSample) this.drainNewestSample = retirementBaselineSample
|
|
866
875
|
|
|
867
876
|
if (this.connectionCount > 0) {
|
|
868
877
|
await new Promise((resolve) => {
|
|
@@ -899,11 +908,21 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
899
908
|
}
|
|
900
909
|
})
|
|
901
910
|
await Promise.allSettled(handoffServices.map((processInstance) => processInstance.stop()))
|
|
911
|
+
|
|
912
|
+
// Last act of the retirement transition, before the release is recorded stopped: whatever
|
|
913
|
+
// is still alive here is the residue, and the bracket pair above can now say whether it is
|
|
914
|
+
// wedged. A generation that stops with a live process group is the state that made
|
|
915
|
+
// operators signal PIDs by hand (STOE-1687), so the transition decides it or reports it —
|
|
916
|
+
// it does not end silently with the process still holding the release.
|
|
917
|
+
if (retirementBaselineSample) await this.escalateWedgedGeneration({previousSample: retirementBaselineSample, reason: "release retirement"})
|
|
918
|
+
|
|
902
919
|
this.state = "stopped"
|
|
903
920
|
this.stoppedAt = new Date().toISOString()
|
|
904
921
|
this.releasePortReservations()
|
|
905
922
|
this.abandonLocalConnections()
|
|
906
|
-
|
|
923
|
+
// The bracket pair is deliberately kept: a generation that stopped with a live process
|
|
924
|
+
// group stays observable, and the deploy path needs the older observation to prove the
|
|
925
|
+
// residue is still flat rather than starting a fresh window from zero.
|
|
907
926
|
}
|
|
908
927
|
|
|
909
928
|
/**
|
|
@@ -938,11 +957,39 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
938
957
|
this.retirementError = undefined
|
|
939
958
|
}
|
|
940
959
|
|
|
941
|
-
/**
|
|
942
|
-
|
|
943
|
-
|
|
960
|
+
/**
|
|
961
|
+
* Stops every process owned by the generation.
|
|
962
|
+
* @param {{retirementTimeoutMs?: number}} [options] - Retirement budget for the processes the
|
|
963
|
+
* supervisor must not kill: a non-blocking-drain process, and any process whose graceful window
|
|
964
|
+
* is `"indefinite"`. Without it those stops have no deadline at all, so an explicit
|
|
965
|
+
* `rollbridge stop` can wait forever — and because that command runs inside an owner mutation,
|
|
966
|
+
* the wait holds the mutation fence and every later deploy is refused (STOE-2015).
|
|
967
|
+
* @returns {Promise<void>} Resolves when stopped, or when retirement stops waiting.
|
|
968
|
+
*/
|
|
969
|
+
async stop(options = {}) {
|
|
970
|
+
const {retirementTimeoutMs} = options
|
|
971
|
+
/** @type {{bounded: boolean, id: string, instance: ManagedProcess, promise: Promise<void>}[]} */
|
|
972
|
+
const stopTasks = [...this.processes.entries()].map(([id, processInstance]) => {
|
|
973
|
+
const bounded = retirementTimeoutMs !== undefined && (this.nonBlockingDrainIds.has(id) || processInstance.hasUnboundedGracefulWindow())
|
|
974
|
+
|
|
975
|
+
return {bounded, id, instance: processInstance, promise: processInstance.stop(bounded ? {retirementTimeoutMs} : {})}
|
|
976
|
+
})
|
|
977
|
+
|
|
978
|
+
await Promise.allSettled(stopTasks.map(({promise}) => promise))
|
|
979
|
+
|
|
980
|
+
// A stop that stopped waiting leaves the process running. Name it, so the operator sees the
|
|
981
|
+
// residue instead of a command that produced nothing.
|
|
982
|
+
for (const {bounded, id, instance} of stopTasks) {
|
|
983
|
+
if (!bounded || !instance.retirementTimedOut) continue
|
|
984
|
+
|
|
985
|
+
this.logger("release stop gave up waiting for a retiring process", {
|
|
986
|
+
pid: instance.status().pid ?? null,
|
|
987
|
+
processId: id,
|
|
988
|
+
releaseId: this.releaseId,
|
|
989
|
+
timeoutMs: retirementTimeoutMs
|
|
990
|
+
})
|
|
991
|
+
}
|
|
944
992
|
|
|
945
|
-
await Promise.allSettled(stopTasks)
|
|
946
993
|
this.state = "stopped"
|
|
947
994
|
this.stoppedAt = new Date().toISOString()
|
|
948
995
|
this.releasePortReservations()
|
|
@@ -969,32 +1016,109 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
969
1016
|
drainStatus() {
|
|
970
1017
|
if (this.state !== "draining") return undefined
|
|
971
1018
|
|
|
972
|
-
|
|
1019
|
+
return this.observeRetirement({nowMs: Date.now()}).classification
|
|
1020
|
+
}
|
|
973
1021
|
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
1022
|
+
/**
|
|
1023
|
+
* Records the newest liveness observation of this generation and classifies its retirement.
|
|
1024
|
+
* @param {object} options - Observation options.
|
|
1025
|
+
* @param {number} options.nowMs - Read time.
|
|
1026
|
+
* @param {DrainLivenessSample | undefined} [options.previousSample] - Explicit older observation to classify against. The retirement transition brackets its own wait with one instead of reusing a stored pair.
|
|
1027
|
+
* @returns {{classification: ReleaseDrainStatus, currentSample: DrainLivenessSample}} The observation the decision acts on, and what the pair says.
|
|
1028
|
+
*/
|
|
1029
|
+
observeRetirement({nowMs, previousSample}) {
|
|
1030
|
+
if (previousSample === undefined && this.connectionCount > 0) {
|
|
1031
|
+
const currentSample = this.drainNewestSample ?? {atMs: nowMs, processes: []}
|
|
1032
|
+
|
|
1033
|
+
return {
|
|
1034
|
+
classification: classifyDrainProgress({
|
|
1035
|
+
connectionCount: this.connectionCount,
|
|
1036
|
+
currentSample,
|
|
1037
|
+
minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
|
|
1038
|
+
previousSample: undefined
|
|
1039
|
+
}),
|
|
1040
|
+
currentSample
|
|
1041
|
+
}
|
|
981
1042
|
}
|
|
982
1043
|
|
|
983
1044
|
const newest = this.drainNewestSample
|
|
984
|
-
const currentSample = newest
|
|
985
|
-
|
|
1045
|
+
const currentSample = previousSample !== undefined || newest === undefined || nowMs - newest.atMs >= DRAIN_SAMPLE_WINDOW_MS
|
|
1046
|
+
? this.takeLivenessSample(nowMs)
|
|
1047
|
+
: newest
|
|
1048
|
+
const priorSample = previousSample ?? (currentSample === newest ? this.drainPriorSample : newest)
|
|
986
1049
|
|
|
987
1050
|
if (currentSample !== newest) {
|
|
988
1051
|
this.drainPriorSample = newest
|
|
989
1052
|
this.drainNewestSample = currentSample
|
|
990
1053
|
}
|
|
991
1054
|
|
|
992
|
-
return
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
1055
|
+
return {
|
|
1056
|
+
classification: classifyDrainProgress({
|
|
1057
|
+
connectionCount: this.connectionCount,
|
|
1058
|
+
currentSample,
|
|
1059
|
+
minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
|
|
1060
|
+
previousSample: priorSample
|
|
1061
|
+
}),
|
|
1062
|
+
currentSample
|
|
1063
|
+
}
|
|
1064
|
+
}
|
|
1065
|
+
|
|
1066
|
+
/**
|
|
1067
|
+
* Applies the wedge escalation to this generation: terminates the processes a proven wedge
|
|
1068
|
+
* proves are holding nothing, and names the ones it refused to terminate.
|
|
1069
|
+
*
|
|
1070
|
+
* Both halves are required. The card this path was written for (STOE-1687) is about a
|
|
1071
|
+
* wedged worker holding a release open for four hours while deploys stack behind it; the
|
|
1072
|
+
* boundary on the same card (STOE-1077) is that no process still holding live work may ever
|
|
1073
|
+
* be SIGKILLed. So a wedge that is not kill-safe is not silently skipped — it is reported
|
|
1074
|
+
* with the pids and process groups it is holding, which is the information an operator needs
|
|
1075
|
+
* in place of guessing.
|
|
1076
|
+
* @param {object} options - Escalation context.
|
|
1077
|
+
* @param {DrainLivenessSample | undefined} [options.previousSample] - Older observation bracketing a wait this caller already performed.
|
|
1078
|
+
* @param {string} options.reason - Named context recorded with the decision.
|
|
1079
|
+
* @returns {Promise<string[]>} Instance ids of the processes the SIGKILL was delivered to.
|
|
1080
|
+
*/
|
|
1081
|
+
async escalateWedgedGeneration({previousSample, reason}) {
|
|
1082
|
+
const {classification, currentSample} = this.observeRetirement({nowMs: Date.now(), previousSample})
|
|
1083
|
+
const decision = decideWedgedEscalation({classification, currentSample, minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS})
|
|
1084
|
+
const liveObservations = currentSample.processes.filter((process) => process.pid !== undefined || process.heldProcessGroupPgid !== undefined)
|
|
1085
|
+
|
|
1086
|
+
if (decision.decision === "report") {
|
|
1087
|
+
if (classification.status === "wedged") {
|
|
1088
|
+
this.logger("release wedged generation left alive", {
|
|
1089
|
+
escalation: "refused",
|
|
1090
|
+
heldProcessGroupPgids: liveObservations.map((process) => process.heldProcessGroupPgid ?? null),
|
|
1091
|
+
pids: liveObservations.map((process) => process.pid ?? null),
|
|
1092
|
+
processIds: liveObservations.map((process) => process.id),
|
|
1093
|
+
reason: decision.reason,
|
|
1094
|
+
releaseId: this.releaseId,
|
|
1095
|
+
trigger: reason
|
|
1096
|
+
})
|
|
1097
|
+
}
|
|
1098
|
+
|
|
1099
|
+
return []
|
|
1100
|
+
}
|
|
1101
|
+
|
|
1102
|
+
const escalated = []
|
|
1103
|
+
|
|
1104
|
+
for (const target of decision.targets) {
|
|
1105
|
+
const processInstance = this.processes.get(target.processId)
|
|
1106
|
+
|
|
1107
|
+
if (!processInstance) continue
|
|
1108
|
+
if (await processInstance.escalateRetirement({reason: `${reason}: ${decision.reason}`})) escalated.push(target.processId)
|
|
1109
|
+
}
|
|
1110
|
+
|
|
1111
|
+
if (escalated.length > 0) {
|
|
1112
|
+
this.logger("release wedged generation escalated", {
|
|
1113
|
+
escalation: "SIGKILL",
|
|
1114
|
+
processIds: escalated,
|
|
1115
|
+
reason: decision.reason,
|
|
1116
|
+
releaseId: this.releaseId,
|
|
1117
|
+
trigger: reason
|
|
1118
|
+
})
|
|
1119
|
+
}
|
|
1120
|
+
|
|
1121
|
+
return escalated
|
|
998
1122
|
}
|
|
999
1123
|
|
|
1000
1124
|
/**
|
|
@@ -59,8 +59,10 @@ test("a Compose unit is normalized with drain-first defaults", () => {
|
|
|
59
59
|
expect(unit.composeFile).toBe("compose.yml")
|
|
60
60
|
expect(unit.stopGracePeriodMs).toBe(10000)
|
|
61
61
|
expect(unit.recreateTimeoutMs).toBe(120000)
|
|
62
|
-
|
|
63
|
-
|
|
62
|
+
// Unset commands are absent from the normalized shape rather than present as undefined, so a unit
|
|
63
|
+
// definition that crossed the guardian's JSON transfer is still equal to a freshly loaded one.
|
|
64
|
+
expect(unit.drain).toEqual({timeoutMs: 60000})
|
|
65
|
+
expect(unit.inFlight).toEqual({intervalMs: 1000, timeoutMs: 0})
|
|
64
66
|
expect(unit.activation).toMatchObject({healthcheck: true, intervalMs: 1000, startDelayMs: 0, timeoutMs: 60000})
|
|
65
67
|
expect(unit.rollback).toEqual({enabled: true})
|
|
66
68
|
expect(config.composeStatePath).toBe(path.resolve("/srv/app", "rollbridge-compose-config-test.compose.state.json"))
|
|
@@ -143,7 +143,9 @@ test("validateConfig defaults the restart policy, accepts overrides, and rejects
|
|
|
143
143
|
const defaulted = validateRestart(undefined)
|
|
144
144
|
|
|
145
145
|
expect(defaulted.issues).toEqual([])
|
|
146
|
-
|
|
146
|
+
// Unset values are absent from the normalized shape rather than present as undefined, so a config
|
|
147
|
+
// that crossed the guardian's JSON transfer is still equal to a freshly loaded one.
|
|
148
|
+
expect(defaulted.config.processes[0].restart).toEqual({backoffFactor: 1, maxDelayMs: 0, windowMs: 0})
|
|
147
149
|
|
|
148
150
|
const custom = validateRestart({backoffFactor: 2, maxDelayMs: 30000, maxRestarts: 5, windowMs: 60000})
|
|
149
151
|
|
|
@@ -705,12 +707,12 @@ test("validateConfig defaults releaseRetention, accepts overrides, and rejects b
|
|
|
705
707
|
const defaulted = validateRetention(undefined)
|
|
706
708
|
|
|
707
709
|
expect(defaulted.issues).toEqual([])
|
|
708
|
-
expect(defaulted.config.releaseRetention).toEqual({keep: 10, maxAgeMs: 0, maxConcurrentDrainingReleaseGroups: 12})
|
|
710
|
+
expect(defaulted.config.releaseRetention).toEqual({escalateWedgedGenerations: false, keep: 10, maxAgeMs: 0, maxConcurrentDrainingReleaseGroups: 12})
|
|
709
711
|
|
|
710
|
-
const custom = validateRetention({keep: 3, maxAgeMs: 60000, maxConcurrentDrainingReleaseGroups: 4})
|
|
712
|
+
const custom = validateRetention({escalateWedgedGenerations: true, keep: 3, maxAgeMs: 60000, maxConcurrentDrainingReleaseGroups: 4})
|
|
711
713
|
|
|
712
714
|
expect(custom.issues).toEqual([])
|
|
713
|
-
expect(custom.config.releaseRetention).toEqual({keep: 3, maxAgeMs: 60000, maxConcurrentDrainingReleaseGroups: 4})
|
|
715
|
+
expect(custom.config.releaseRetention).toEqual({escalateWedgedGenerations: true, keep: 3, maxAgeMs: 60000, maxConcurrentDrainingReleaseGroups: 4})
|
|
714
716
|
|
|
715
717
|
// 0 is a valid bound: it disables the pre-activation draining-generation check.
|
|
716
718
|
const unbounded = validateRetention({maxConcurrentDrainingReleaseGroups: 0})
|
|
@@ -718,8 +720,9 @@ test("validateConfig defaults releaseRetention, accepts overrides, and rejects b
|
|
|
718
720
|
expect(unbounded.issues).toEqual([])
|
|
719
721
|
expect(unbounded.config.releaseRetention.maxConcurrentDrainingReleaseGroups).toBe(0)
|
|
720
722
|
|
|
721
|
-
const invalid = validateRetention({keep: -1, maxAgeMs: -5, maxConcurrentDrainingReleaseGroups: -2})
|
|
723
|
+
const invalid = validateRetention({escalateWedgedGenerations: "yes", keep: -1, maxAgeMs: -5, maxConcurrentDrainingReleaseGroups: -2})
|
|
722
724
|
|
|
725
|
+
expect(invalid.issues.some((issue) => issue.message === "releaseRetention.escalateWedgedGenerations must be a boolean")).toBeTruthy()
|
|
723
726
|
expect(invalid.issues.some((issue) => issue.message === "releaseRetention.keep must be a non-negative integer")).toBeTruthy()
|
|
724
727
|
expect(invalid.issues.some((issue) => issue.message === "releaseRetention.maxAgeMs must be a non-negative number")).toBeTruthy()
|
|
725
728
|
expect(invalid.issues.some((issue) => issue.message === "releaseRetention.maxConcurrentDrainingReleaseGroups must be a non-negative integer")).toBeTruthy()
|
|
@@ -770,6 +773,24 @@ test("ownerRecovery requires durable state and a non-negative integer reconnecti
|
|
|
770
773
|
expect(invalidGrace.issues.some((issue) => issue.message === "ownerRecovery.reconnectGraceMs must be a non-negative integer")).toBeTruthy()
|
|
771
774
|
})
|
|
772
775
|
|
|
776
|
+
test("a normalized config is identical to its own JSON form, so the guardian transfer cannot change it", () => {
|
|
777
|
+
const {config, issues} = validateConfig({
|
|
778
|
+
application: "demo",
|
|
779
|
+
control: {path: "/tmp/demo.sock"},
|
|
780
|
+
processes: [{command: "run web", id: "web", policy: "proxied", port: {from: 18000, to: 18099}}],
|
|
781
|
+
proxy: {host: "127.0.0.1", port: 8182}
|
|
782
|
+
}, "/srv/app/rollbridge.js")
|
|
783
|
+
|
|
784
|
+
expect(issues).toEqual([])
|
|
785
|
+
|
|
786
|
+
// Owner replacement publishes release configs to the candidate daemon over the guardian's JSON
|
|
787
|
+
// socket, which drops undefined-valued properties. A config that is not its own JSON form therefore
|
|
788
|
+
// arrives at the candidate unequal to the config file it was started from, and the live-reload fence
|
|
789
|
+
// aborts the next deploy with "Config changes to control cannot be applied live" even though the
|
|
790
|
+
// file never changed — measured in production on 2026-11-16 (tensorbuzz/production).
|
|
791
|
+
expect(config).toEqual(JSON.parse(JSON.stringify(config)))
|
|
792
|
+
})
|
|
793
|
+
|
|
773
794
|
test("normalizeConfig throws an aggregated error listing every issue", async () => {
|
|
774
795
|
const normalization = Promise.resolve().then(() => normalizeConfig({
|
|
775
796
|
application: "demo",
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
3
|
import {describe, expect, test} from "@velocious/testing"
|
|
4
|
-
import {admitGenerationRelease, classifyDrainProgress} from "../src/drain-policy.js"
|
|
4
|
+
import {admitGenerationRelease, classifyDrainProgress, decideWedgedEscalation} from "../src/drain-policy.js"
|
|
5
5
|
|
|
6
6
|
describe("drain-policy", () => {
|
|
7
7
|
|
|
@@ -47,6 +47,23 @@ function classify(options = {}) {
|
|
|
47
47
|
})
|
|
48
48
|
}
|
|
49
49
|
|
|
50
|
+
/**
|
|
51
|
+
* @param {object} [options] - Decision inputs.
|
|
52
|
+
* @param {number} [options.connectionCount] - Live connections owned by the generation.
|
|
53
|
+
* @param {import("../src/drain-policy.js").DrainLivenessSample} [options.current] - Newest observation.
|
|
54
|
+
* @param {import("../src/drain-policy.js").DrainLivenessSample | undefined} [options.previous] - Older observation.
|
|
55
|
+
* @returns {import("../src/drain-policy.js").EscalationDecision} Escalation decision for the same pair the classifier saw.
|
|
56
|
+
*/
|
|
57
|
+
function escalate(options = {}) {
|
|
58
|
+
const currentSample = options.current ?? sample()
|
|
59
|
+
|
|
60
|
+
return decideWedgedEscalation({
|
|
61
|
+
classification: classify({connectionCount: options.connectionCount, current: currentSample, previous: options.previous}),
|
|
62
|
+
currentSample,
|
|
63
|
+
minSampleWindowMs: WINDOW_MS
|
|
64
|
+
})
|
|
65
|
+
}
|
|
66
|
+
|
|
50
67
|
test("a generation that still owns connections is draining, never wedged", () => {
|
|
51
68
|
const result = classify({connectionCount: 3, current: sample({liveChildCount: 0})})
|
|
52
69
|
|
|
@@ -151,6 +168,90 @@ test("a held process group that still consumes CPU across the window is a progre
|
|
|
151
168
|
expect(result.reason).toBe("CPU advanced 4000ms across 25000ms")
|
|
152
169
|
})
|
|
153
170
|
|
|
171
|
+
test("a proven wedge with no runner child left is escalated, naming the process group it terminates", () => {
|
|
172
|
+
const previous = sample({atMs: 1000000, cpuMs: 5000, liveChildCount: 3})
|
|
173
|
+
const current = sample({atMs: 1025000, cpuMs: 5000, liveChildCount: 0})
|
|
174
|
+
const decision = escalate({current, previous})
|
|
175
|
+
|
|
176
|
+
expect(decision.decision).toBe("escalate")
|
|
177
|
+
expect(decision.killSafe).toBe(true)
|
|
178
|
+
expect(decision.targets).toEqual([{pgid: 4242, processId: "worker"}])
|
|
179
|
+
expect(decision.reason).toContain("CPU flat across 25000ms")
|
|
180
|
+
})
|
|
181
|
+
|
|
182
|
+
test("the escalation targets the group that outlived its tracked process", () => {
|
|
183
|
+
const previous = sample({atMs: 1000000, cpuMs: 5000, heldProcessGroupPgid: 4242, liveChildCount: 2, pid: undefined})
|
|
184
|
+
const current = sample({atMs: 1025000, cpuMs: 5000, heldProcessGroupPgid: 4242, liveChildCount: 0, pid: undefined})
|
|
185
|
+
const decision = escalate({current, previous})
|
|
186
|
+
|
|
187
|
+
expect(decision.decision).toBe("escalate")
|
|
188
|
+
expect(decision.targets).toEqual([{pgid: 4242, processId: "worker"}])
|
|
189
|
+
})
|
|
190
|
+
|
|
191
|
+
test("a wedge that still holds a live runner child is never escalated", () => {
|
|
192
|
+
const previous = sample({atMs: 1000000, cpuMs: 5000})
|
|
193
|
+
const current = sample({atMs: 1025000, cpuMs: 5000, liveChildCount: 3})
|
|
194
|
+
|
|
195
|
+
expect(classify({current, previous}).status).toBe("wedged")
|
|
196
|
+
|
|
197
|
+
const decision = escalate({current, previous})
|
|
198
|
+
|
|
199
|
+
expect(decision.decision).toBe("report")
|
|
200
|
+
expect(decision.killSafe).toBe(false)
|
|
201
|
+
expect(decision.targets).toEqual([])
|
|
202
|
+
expect(decision.reason).toBe("no escalation: 3 runner children are still live in the generation; SIGKILL would abort in-flight work")
|
|
203
|
+
})
|
|
204
|
+
|
|
205
|
+
test("a live process whose CPU is still moving is never escalated", () => {
|
|
206
|
+
const previous = sample({atMs: 1000000, cpuMs: 5000})
|
|
207
|
+
const current = sample({atMs: 1025000, cpuMs: 9000, liveChildCount: 0})
|
|
208
|
+
|
|
209
|
+
expect(classify({current, previous}).status).toBe("wedged")
|
|
210
|
+
|
|
211
|
+
const decision = escalate({current, previous})
|
|
212
|
+
|
|
213
|
+
expect(decision.decision).toBe("report")
|
|
214
|
+
expect(decision.reason).toBe("no escalation: CPU flatness is not proven (4000ms of CPU moved across 25000ms)")
|
|
215
|
+
})
|
|
216
|
+
|
|
217
|
+
test("a process that has not expired its retirement budget is never escalated", () => {
|
|
218
|
+
const previous = sample({atMs: 1000000, cpuMs: 5000, liveChildCount: 0, retirementTimedOut: false})
|
|
219
|
+
const current = sample({atMs: 1025000, cpuMs: 5000, liveChildCount: 0, retirementTimedOut: false})
|
|
220
|
+
const decision = escalate({current, previous})
|
|
221
|
+
|
|
222
|
+
expect(decision.decision).toBe("report")
|
|
223
|
+
expect(decision.reason).toBe("no escalation: drain is progressing (process worker is still inside its retirement budget)")
|
|
224
|
+
})
|
|
225
|
+
|
|
226
|
+
test("a generation that still owns a live connection is never escalated", () => {
|
|
227
|
+
const decision = escalate({connectionCount: 2, current: sample({liveChildCount: 0})})
|
|
228
|
+
|
|
229
|
+
expect(decision.decision).toBe("report")
|
|
230
|
+
expect(decision.reason).toBe("no escalation: drain is progressing (2 live connections still owned by the generation)")
|
|
231
|
+
})
|
|
232
|
+
|
|
233
|
+
test("a wedge with no measurable CPU is never escalated", () => {
|
|
234
|
+
const previous = sample({atMs: 1000000, cpuMs: undefined, liveChildCount: 0})
|
|
235
|
+
const current = sample({atMs: 1025000, cpuMs: undefined, liveChildCount: 0})
|
|
236
|
+
|
|
237
|
+
expect(classify({current, previous}).status).toBe("wedged")
|
|
238
|
+
|
|
239
|
+
const decision = escalate({current, previous})
|
|
240
|
+
|
|
241
|
+
expect(decision.decision).toBe("report")
|
|
242
|
+
expect(decision.reason).toBe("no escalation: CPU flatness is not proven (process CPU is not measurable on this host)")
|
|
243
|
+
})
|
|
244
|
+
|
|
245
|
+
test("a wedge seen in one observation is not escalated until a full window has passed", () => {
|
|
246
|
+
const current = sample({liveChildCount: 0})
|
|
247
|
+
const shortWindow = escalate({current: sample({atMs: 1001000, cpuMs: 5000, liveChildCount: 0}), previous: sample({atMs: 1000000, cpuMs: 5000, liveChildCount: 0})})
|
|
248
|
+
|
|
249
|
+
expect(classify({current}).status).toBe("wedged")
|
|
250
|
+
expect(escalate({current}).decision).toBe("report")
|
|
251
|
+
expect(escalate({current}).reason).toBe("no escalation: liveness window 0ms is below the 20000ms minimum")
|
|
252
|
+
expect(shortWindow.reason).toBe("no escalation: liveness window 1000ms is below the 20000ms minimum")
|
|
253
|
+
})
|
|
254
|
+
|
|
154
255
|
test("admission allows a handoff that stays within the bound and counts the incumbent", () => {
|
|
155
256
|
const admission = admitGenerationRelease({
|
|
156
257
|
candidateReleaseId: "v3",
|
|
@@ -309,10 +309,12 @@ test("runs quiet and drain lifecycle hooks before stopping", async () => {
|
|
|
309
309
|
})
|
|
310
310
|
|
|
311
311
|
test("retirement timeout leaves an indefinite worker alive without SIGKILL", async () => {
|
|
312
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-retirement-wedge-"))
|
|
313
|
+
const readyPath = path.join(dir, "ready")
|
|
312
314
|
/** @type {string[]} */
|
|
313
315
|
const signals = []
|
|
314
316
|
const managed = new ManagedProcess({
|
|
315
|
-
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(
|
|
317
|
+
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`process.on('SIGTERM', () => {}); require("node:fs").writeFileSync(${JSON.stringify(readyPath)}, "ready"); setInterval(() => {}, 1000)`)}`,
|
|
316
318
|
cwd: undefined,
|
|
317
319
|
env: {},
|
|
318
320
|
id: "worker",
|
|
@@ -332,8 +334,216 @@ test("retirement timeout leaves an indefinite worker alive without SIGKILL", asy
|
|
|
332
334
|
|
|
333
335
|
try {
|
|
334
336
|
await managed.start()
|
|
337
|
+
// The fixture installs its SIGTERM disposition before it announces readiness. Signalling a
|
|
338
|
+
// child that has not finished booting reaches the default disposition instead, which exits
|
|
339
|
+
// the group and leaves no wedge to observe.
|
|
340
|
+
await waitFor(() => fs.existsSync(readyPath))
|
|
341
|
+
await managed.stop({retirementTimeoutMs: 25})
|
|
342
|
+
|
|
343
|
+
expect(managed.retirementTimedOut).toBe(true)
|
|
344
|
+
expect(signals).toEqual(["SIGTERM"])
|
|
345
|
+
expect(managed.status().pid).toBeDefined()
|
|
346
|
+
} finally {
|
|
347
|
+
const pid = managed.status().pid
|
|
348
|
+
if (pid) process.kill(-pid, "SIGKILL")
|
|
349
|
+
if (managed.exitPromise) await managed.exitPromise
|
|
350
|
+
fs.rmSync(dir, {force: true, recursive: true})
|
|
351
|
+
}
|
|
352
|
+
})
|
|
353
|
+
|
|
354
|
+
test("escalateRetirement terminates the wedged worker the bounded retirement left alive", async () => {
|
|
355
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-escalation-wedge-"))
|
|
356
|
+
const readyPath = path.join(dir, "ready")
|
|
357
|
+
/** @type {string[]} */
|
|
358
|
+
const signals = []
|
|
359
|
+
/** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
|
|
360
|
+
const logs = []
|
|
361
|
+
const managed = new ManagedProcess({
|
|
362
|
+
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`process.on('SIGTERM', () => {}); require("node:fs").writeFileSync(${JSON.stringify(readyPath)}, "ready"); setInterval(() => {}, 1000)`)}`,
|
|
363
|
+
cwd: undefined,
|
|
364
|
+
env: {},
|
|
365
|
+
id: "worker",
|
|
366
|
+
lifecycle: {drainTimeoutMs: 0},
|
|
367
|
+
logger: (message, data) => logs.push({data, message}),
|
|
368
|
+
outputLines: 50,
|
|
369
|
+
restartDelayMs: 10,
|
|
370
|
+
shouldRestart: () => false,
|
|
371
|
+
stopSignal: "SIGTERM",
|
|
372
|
+
stopTimeoutMs: "indefinite"
|
|
373
|
+
})
|
|
374
|
+
const signalProcessGroup = managed.signalProcessGroup.bind(managed)
|
|
375
|
+
|
|
376
|
+
managed.signalProcessGroup = async (signal, pgid, deadline) => {
|
|
377
|
+
signals.push(signal)
|
|
378
|
+
await signalProcessGroup(signal, pgid, deadline)
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
try {
|
|
382
|
+
await managed.start()
|
|
383
|
+
await waitFor(() => fs.existsSync(readyPath))
|
|
384
|
+
await managed.stop({retirementTimeoutMs: 25})
|
|
385
|
+
const wedgedPid = managed.status().pid
|
|
386
|
+
|
|
387
|
+
expect(managed.retirementTimedOut).toBe(true)
|
|
388
|
+
expect(signals).toEqual(["SIGTERM"])
|
|
389
|
+
expect(wedgedPid).toBeDefined()
|
|
390
|
+
|
|
391
|
+
expect(await managed.escalateRetirement({reason: "spec: proven wedge"})).toBe(true)
|
|
392
|
+
expect(signals).toEqual(["SIGTERM", "SIGKILL"])
|
|
393
|
+
expect(logs.some((entry) => entry.message === "process retirement escalated to SIGKILL" && entry.data?.reason === "spec: proven wedge")).toBe(true)
|
|
394
|
+
expect(await managed.waitForProcessGroupExit(/** @type {number} */ (wedgedPid), Date.now() + 5000)).toBe(true)
|
|
395
|
+
if (managed.exitPromise) await managed.exitPromise
|
|
396
|
+
expect(managed.status().pid).toBeUndefined()
|
|
397
|
+
expect(managed.status().state).toBe("stopped")
|
|
398
|
+
} finally {
|
|
399
|
+
const pid = managed.status().pid
|
|
400
|
+
if (pid) process.kill(-pid, "SIGKILL")
|
|
401
|
+
if (managed.exitPromise) await managed.exitPromise
|
|
402
|
+
fs.rmSync(dir, {force: true, recursive: true})
|
|
403
|
+
}
|
|
404
|
+
})
|
|
405
|
+
|
|
406
|
+
test("escalateRetirement refuses a process that has not expired its retirement budget", async () => {
|
|
407
|
+
/** @type {string[]} */
|
|
408
|
+
const signals = []
|
|
409
|
+
/** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
|
|
410
|
+
const logs = []
|
|
411
|
+
const managed = new ManagedProcess({
|
|
412
|
+
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify("setInterval(() => {}, 1000)")}`,
|
|
413
|
+
cwd: undefined,
|
|
414
|
+
env: {},
|
|
415
|
+
id: "worker",
|
|
416
|
+
lifecycle: {drainTimeoutMs: 0},
|
|
417
|
+
logger: (message, data) => logs.push({data, message}),
|
|
418
|
+
outputLines: 50,
|
|
419
|
+
restartDelayMs: 10,
|
|
420
|
+
shouldRestart: () => false,
|
|
421
|
+
stopSignal: "SIGTERM",
|
|
422
|
+
stopTimeoutMs: "indefinite"
|
|
423
|
+
})
|
|
424
|
+
const signalProcessGroup = managed.signalProcessGroup.bind(managed)
|
|
425
|
+
|
|
426
|
+
managed.signalProcessGroup = async (signal, pgid, deadline) => {
|
|
427
|
+
signals.push(signal)
|
|
428
|
+
await signalProcessGroup(signal, pgid, deadline)
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
try {
|
|
432
|
+
await managed.start()
|
|
433
|
+
await managed.quiesce()
|
|
434
|
+
|
|
435
|
+
expect(managed.retirementTimedOut).toBe(false)
|
|
436
|
+
expect(await managed.escalateRetirement({reason: "spec"})).toBe(false)
|
|
437
|
+
expect(signals).toEqual([])
|
|
438
|
+
expect(logs.some((entry) => entry.message === "retirement escalation refused; process is not past its retirement budget")).toBe(true)
|
|
439
|
+
expect(managed.status().pid).toBeDefined()
|
|
440
|
+
} finally {
|
|
441
|
+
const pid = managed.status().pid
|
|
442
|
+
if (pid) process.kill(-pid, "SIGKILL")
|
|
443
|
+
if (managed.exitPromise) await managed.exitPromise
|
|
444
|
+
}
|
|
445
|
+
})
|
|
446
|
+
|
|
447
|
+
test("escalateRetirement is inert for a process that drained normally", async () => {
|
|
448
|
+
/** @type {string[]} */
|
|
449
|
+
const signals = []
|
|
450
|
+
const managed = new ManagedProcess({
|
|
451
|
+
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify("setInterval(() => {}, 1000)")}`,
|
|
452
|
+
cwd: undefined,
|
|
453
|
+
env: {},
|
|
454
|
+
id: "worker",
|
|
455
|
+
lifecycle: {drainTimeoutMs: 0},
|
|
456
|
+
logger: () => {},
|
|
457
|
+
outputLines: 50,
|
|
458
|
+
restartDelayMs: 10,
|
|
459
|
+
shouldRestart: () => false,
|
|
460
|
+
stopSignal: "SIGTERM",
|
|
461
|
+
stopTimeoutMs: 2000
|
|
462
|
+
})
|
|
463
|
+
const signalProcessGroup = managed.signalProcessGroup.bind(managed)
|
|
464
|
+
|
|
465
|
+
managed.signalProcessGroup = async (signal, pgid, deadline) => {
|
|
466
|
+
signals.push(signal)
|
|
467
|
+
await signalProcessGroup(signal, pgid, deadline)
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
try {
|
|
471
|
+
await managed.start()
|
|
472
|
+
await managed.stop()
|
|
473
|
+
|
|
474
|
+
expect(managed.status().state).toBe("stopped")
|
|
475
|
+
expect(await managed.escalateRetirement({reason: "spec"})).toBe(false)
|
|
476
|
+
expect(signals).toEqual(["SIGTERM"])
|
|
477
|
+
} finally {
|
|
478
|
+
const pid = managed.status().pid
|
|
479
|
+
if (pid) process.kill(-pid, "SIGKILL")
|
|
480
|
+
if (managed.exitPromise) await managed.exitPromise
|
|
481
|
+
}
|
|
482
|
+
})
|
|
483
|
+
|
|
484
|
+
test("a retirement budget caps a finite graceful window instead of being outvoted by it", async () => {
|
|
485
|
+
/** @type {string[]} */
|
|
486
|
+
const signals = []
|
|
487
|
+
/** @type {Array<{msFromStopStart: number, phase: string}>} */
|
|
488
|
+
const deadlines = []
|
|
489
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-retirement-bound-"))
|
|
490
|
+
const readyPath = path.join(dir, "ready")
|
|
491
|
+
const managed = new ManagedProcess({
|
|
492
|
+
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`process.on('SIGTERM', () => {}); require("node:fs").writeFileSync(${JSON.stringify(readyPath)}, "ready"); setInterval(() => {}, 1000)`)}`,
|
|
493
|
+
cwd: undefined,
|
|
494
|
+
env: {},
|
|
495
|
+
id: "worker",
|
|
496
|
+
lifecycle: {drainTimeoutMs: 0},
|
|
497
|
+
logger: () => {},
|
|
498
|
+
outputLines: 50,
|
|
499
|
+
restartDelayMs: 10,
|
|
500
|
+
shouldRestart: () => false,
|
|
501
|
+
stopSignal: "SIGTERM",
|
|
502
|
+
stopTimeoutMs: 2000
|
|
503
|
+
})
|
|
504
|
+
let stopStartedAtMs = 0
|
|
505
|
+
|
|
506
|
+
/**
|
|
507
|
+
* @param {string} phase - The step the deadline was handed to.
|
|
508
|
+
* @param {number | undefined} deadline - Absolute deadline the step received.
|
|
509
|
+
* @returns {void} Records the deadline as an offset from the start of the stop.
|
|
510
|
+
*/
|
|
511
|
+
const recordDeadline = (phase, deadline) => {
|
|
512
|
+
if (deadline !== undefined) deadlines.push({msFromStopStart: deadline - stopStartedAtMs, phase})
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
const signalProcessGroup = managed.signalProcessGroup.bind(managed)
|
|
516
|
+
managed.signalProcessGroup = async (signal, pgid, deadline) => {
|
|
517
|
+
signals.push(signal)
|
|
518
|
+
recordDeadline("signal", deadline)
|
|
519
|
+
await signalProcessGroup(signal, pgid, deadline)
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
const waitForProcessGroupExit = managed.waitForProcessGroupExit.bind(managed)
|
|
523
|
+
managed.waitForProcessGroupExit = async (pgid, deadline) => {
|
|
524
|
+
recordDeadline("wait", deadline)
|
|
525
|
+
return await waitForProcessGroupExit(pgid, deadline)
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
try {
|
|
529
|
+
await managed.start()
|
|
530
|
+
// The fixture installs its SIGTERM disposition before it announces readiness. Signalling a
|
|
531
|
+
// child that has not finished booting reaches the default disposition instead, which exits
|
|
532
|
+
// the group and leaves no retirement wait to observe.
|
|
533
|
+
await waitFor(() => fs.existsSync(readyPath))
|
|
534
|
+
stopStartedAtMs = Date.now()
|
|
535
|
+
|
|
335
536
|
await managed.stop({retirementTimeoutMs: 25})
|
|
336
537
|
|
|
538
|
+
// The 2 000 ms graceful window must not outvote the 25 ms retirement budget. Read the other
|
|
539
|
+
// way round the wait runs the whole window before it escalates to SIGKILL, which is the shape
|
|
540
|
+
// that held the TensorBuzz control plane for 4,5 h after the worker moved from "indefinite" to
|
|
541
|
+
// 4,5 h and killed the in-flight builds the consumer said not to interrupt.
|
|
542
|
+
expect(deadlines.map((entry) => entry.phase)).toEqual(["signal", "wait"])
|
|
543
|
+
for (const entry of deadlines) {
|
|
544
|
+
expect(entry.msFromStopStart).toBeGreaterThanOrEqual(25)
|
|
545
|
+
expect(entry.msFromStopStart).toBeLessThan(1000)
|
|
546
|
+
}
|
|
337
547
|
expect(managed.retirementTimedOut).toBe(true)
|
|
338
548
|
expect(signals).toEqual(["SIGTERM"])
|
|
339
549
|
expect(managed.status().pid).toBeDefined()
|
|
@@ -341,6 +551,7 @@ test("retirement timeout leaves an indefinite worker alive without SIGKILL", asy
|
|
|
341
551
|
const pid = managed.status().pid
|
|
342
552
|
if (pid) process.kill(-pid, "SIGKILL")
|
|
343
553
|
if (managed.exitPromise) await managed.exitPromise
|
|
554
|
+
fs.rmSync(dir, {force: true, recursive: true})
|
|
344
555
|
}
|
|
345
556
|
})
|
|
346
557
|
|