rollbridge 0.1.73 → 0.1.75
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +63 -10
- package/TODO.md +13 -0
- package/changelog.d/20261025-guardian-re-register-dead-generation.md +13 -0
- package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +1 -0
- package/changelog.d/20261104-compose-service-units.md +21 -0
- package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +20 -0
- package/changelog.d/20261106-release-dependency-boot-budgets.md +32 -0
- package/changelog.d/20261109-path-filtered-builds.md +11 -0
- package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +26 -0
- package/changelog.d/20261111-contention-sensitive-daemon-specs.md +27 -0
- package/changelog.d/20261113-bounded-nonblocking-retirement.md +5 -0
- package/changelog.d/20261115-bound-draining-release-groups.md +62 -0
- package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +27 -0
- package/changelog.d/20261115-wedged-generation-escalation.md +44 -0
- package/changelog.d/20261116-bounded-explicit-stop-retirement.md +30 -0
- package/changelog.d/20261116-config-json-canonical-shape.md +11 -0
- package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +29 -0
- package/changelog.d/20261116-guardian-version-handoff.md +37 -0
- package/changelog.d/20261116-reclaim-dead-owner-mutation.md +7 -0
- package/changelog.d/20261117-activate-before-incumbent-retirement.md +1 -0
- package/changelog.d/20261117-activation-path-drain-deadline.md +35 -0
- package/changelog.d/20261117-stop-companion-after-retired-dependency.md +7 -0
- package/changelog.d/20261118-bounded-control-client-close.md +4 -0
- package/changelog.d/20261118-incumbent-retirement-authority-guard.md +1 -0
- package/changelog.d/20261118-owner-retirement-no-guardian-bound.md +19 -0
- package/changelog.d/20261118-shutdown-retirement-bound-restored.md +31 -0
- package/docs/cli.md +128 -3
- package/docs/compose-services.md +222 -0
- package/docs/config.md +177 -6
- package/docs/troubleshooting.md +97 -6
- package/docs/velocious.md +1 -1
- package/docs/workers.md +11 -0
- package/examples/tensorbuzz.com.js +3 -2
- package/package.json +1 -1
- package/src/cli.js +69 -3
- package/src/compose-unit.js +725 -0
- package/src/config.js +346 -8
- package/src/control-client.js +32 -5
- package/src/daemon.js +448 -76
- package/src/doctor.js +74 -0
- package/src/drain-policy.js +252 -0
- package/src/guardian-client.js +92 -9
- package/src/health.js +99 -8
- package/src/managed-process.js +197 -16
- package/src/process-guardian.js +212 -22
- package/src/process-memory.js +62 -2
- package/src/process-order.js +26 -0
- package/src/release-group.js +452 -39
- package/tensorbuzz.yml +20 -1
- package/test/completion.test.js +5 -2
- package/test/compose-config.test.js +160 -0
- package/test/compose-docker.test.js +297 -0
- package/test/compose-unit.test.js +415 -0
- package/test/config-validation.test.js +135 -5
- package/test/control-protocol.test.js +22 -0
- package/test/drain-policy.test.js +328 -0
- package/test/fixtures/deferred-boot-app.js +106 -0
- package/test/fixtures/gated-app.js +50 -0
- package/test/guardian-client.test.js +149 -5
- package/test/health.test.js +74 -1
- package/test/managed-process.test.js +260 -1
- package/test/owner-recovery.test.js +151 -20
- package/test/owner-replacement.test.js +176 -15
- package/test/process-memory.test.js +47 -1
- package/test/release-group.test.js +447 -2
- package/test/rollbridge.test.js +774 -72
- package/test/shutdown-completion.test.js +119 -0
- package/test/shutdown-retirement-bound.test.js +162 -0
- package/test/support/process.js +3 -1
- package/test/tensorbuzz-config.test.js +228 -0
package/src/doctor.js
CHANGED
|
@@ -5,6 +5,7 @@ import fs from "node:fs/promises"
|
|
|
5
5
|
import net from "node:net"
|
|
6
6
|
import path from "node:path"
|
|
7
7
|
import {inspectControlSocket} from "./daemon.js"
|
|
8
|
+
import {composeGlobalArgs, runComposeCommand} from "./compose-unit.js"
|
|
8
9
|
import {liveProcesses, readState} from "./state-store.js"
|
|
9
10
|
import {processTemplateContext, renderObject, renderTemplate} from "./template.js"
|
|
10
11
|
|
|
@@ -25,6 +26,13 @@ export async function runEnvironmentChecks(config) {
|
|
|
25
26
|
const socketInspection = await inspectControlSocketSafely(config.control.path)
|
|
26
27
|
|
|
27
28
|
checks.push(controlSocketCheck(config, socketInspection))
|
|
29
|
+
if (!("error" in socketInspection) && socketInspection.guardian) {
|
|
30
|
+
const {guardian} = socketInspection
|
|
31
|
+
const lease = guardian.ownerMutation
|
|
32
|
+
const leaseDetail = lease ? `, owner mutation ${String(lease.operation || "unnamed operation")} held by pid ${String(lease.pid || "unknown")}` : ", no active owner-mutation lease"
|
|
33
|
+
|
|
34
|
+
checks.push({detail: `guardian Rollbridge ${guardian.version || "unknown"}${leaseDetail}`, name: "guardian diagnostics", ok: guardian.version !== null})
|
|
35
|
+
}
|
|
28
36
|
checks.push(await controlSocketDirectoryCheck(config))
|
|
29
37
|
checks.push(await proxyPortCheck(config, socketInspection))
|
|
30
38
|
|
|
@@ -36,9 +44,75 @@ export async function runEnvironmentChecks(config) {
|
|
|
36
44
|
checks.push(await orphanCheck(config.statePath, daemonRunning))
|
|
37
45
|
}
|
|
38
46
|
|
|
47
|
+
if (config.compose !== undefined) checks.push(...await runComposeChecks(config))
|
|
48
|
+
|
|
49
|
+
return checks
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Pre-flights every declared Compose unit: the Docker Compose CLI, the unit's Compose file, and
|
|
54
|
+
* the service it names inside that file.
|
|
55
|
+
* @param {import("./config.js").RollbridgeConfig} config - Normalized config.
|
|
56
|
+
* @returns {Promise<DoctorCheck[]>} One check per probed aspect.
|
|
57
|
+
*/
|
|
58
|
+
async function runComposeChecks(config) {
|
|
59
|
+
/** @type {DoctorCheck[]} */
|
|
60
|
+
const checks = []
|
|
61
|
+
const version = await runComposeCommand({args: ["docker", "compose", "version"], timeoutMs: 15000})
|
|
62
|
+
|
|
63
|
+
if (version.code !== 0) {
|
|
64
|
+
return [{detail: `docker compose is not usable: ${version.stderr.trim() || `exit ${version.code}`}`, name: "docker compose", ok: false}]
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
checks.push({detail: version.stdout.trim() || "available", name: "docker compose", ok: true})
|
|
68
|
+
|
|
69
|
+
for (const unit of config.compose ?? []) {
|
|
70
|
+
const composeFile = path.resolve(unit.projectDirectory, unit.composeFile)
|
|
71
|
+
|
|
72
|
+
if (!await fileExists(composeFile)) {
|
|
73
|
+
checks.push({detail: `${composeFile} is missing; unit "${unit.id}" cannot be rolled`, name: `compose unit ${unit.id}`, ok: false})
|
|
74
|
+
continue
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
if (unit.envFile !== undefined && !await fileExists(path.resolve(unit.projectDirectory, unit.envFile))) {
|
|
78
|
+
checks.push({detail: `${path.resolve(unit.projectDirectory, unit.envFile)} is missing; unit "${unit.id}" cannot be rolled`, name: `compose unit ${unit.id}`, ok: false})
|
|
79
|
+
continue
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
const services = await runComposeCommand({args: [...composeGlobalArgs(unit), "config", "--services"], cwd: unit.projectDirectory, timeoutMs: 30000})
|
|
83
|
+
|
|
84
|
+
if (services.code !== 0) {
|
|
85
|
+
checks.push({detail: `could not read ${composeFile}: ${services.stderr.trim() || `exit ${services.code}`}`, name: `compose unit ${unit.id}`, ok: false})
|
|
86
|
+
continue
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
const listed = services.stdout.split("\n").map((line) => line.trim()).filter(Boolean)
|
|
90
|
+
|
|
91
|
+
if (!listed.includes(unit.service)) {
|
|
92
|
+
checks.push({detail: `service "${unit.service}" is not declared in ${composeFile}; found ${listed.join(", ") || "no services"}`, name: `compose unit ${unit.id}`, ok: false})
|
|
93
|
+
continue
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
checks.push({detail: `${unit.service} in ${composeFile}, target image ${unit.image}`, name: `compose unit ${unit.id}`, ok: true})
|
|
97
|
+
}
|
|
98
|
+
|
|
39
99
|
return checks
|
|
40
100
|
}
|
|
41
101
|
|
|
102
|
+
/**
|
|
103
|
+
* @param {string} filePath - Path to probe.
|
|
104
|
+
* @returns {Promise<boolean>} True when the path exists.
|
|
105
|
+
*/
|
|
106
|
+
async function fileExists(filePath) {
|
|
107
|
+
try {
|
|
108
|
+
await fs.access(filePath)
|
|
109
|
+
|
|
110
|
+
return true
|
|
111
|
+
} catch {
|
|
112
|
+
return false
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
|
|
42
116
|
/**
|
|
43
117
|
* @param {string} statePath - Configured state file path.
|
|
44
118
|
* @returns {Promise<DoctorCheck>} Whether the state file's directory is writable.
|
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* @typedef {"progressing" | "timedOut" | "wedged" | "unproven"} DrainProgress
|
|
5
|
+
* @typedef {{cpuMs: number | undefined, heldProcessGroupPgid: number | undefined, id: string, liveChildCount: number | undefined, pid: number | undefined, retirementTimedOut: boolean}} DrainProcessObservation
|
|
6
|
+
* @typedef {{atMs: number, processes: DrainProcessObservation[]}} DrainLivenessSample
|
|
7
|
+
* @typedef {{cpuDeltaMs: number | undefined, reason: string, sampleWindowMs: number, status: DrainProgress}} DrainClassification
|
|
8
|
+
* @typedef {{releaseId: string, reason: string, status: DrainProgress}} DrainGroupReport
|
|
9
|
+
* @typedef {{allowed: boolean, draining: DrainGroupReport[], reason: string | undefined, wouldBeDrainingCount: number}} GenerationAdmission
|
|
10
|
+
* @typedef {{pgid: number, processId: string}} EscalationTarget
|
|
11
|
+
* @typedef {{decision: "escalate" | "report", killSafe: boolean, reason: string, targets: EscalationTarget[]}} EscalationDecision
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Classifies one draining release group as still draining, past its drain budget, or wedged.
|
|
16
|
+
*
|
|
17
|
+
* A group is wedged only when its drain can no longer progress. Every inconclusive reading
|
|
18
|
+
* — a live connection, a process still inside its retirement budget, an unmeasurable CPU
|
|
19
|
+
* counter, or a sample window shorter than `minSampleWindowMs` — reports `progressing` or
|
|
20
|
+
* `unproven`, never `wedged`: a legitimate multi-hour drain is valid and must not be
|
|
21
|
+
* mislabelled as a wedge.
|
|
22
|
+
*
|
|
23
|
+
* A live connection inside `drainBudgetMs` is still a legitimate drain. Past that budget the
|
|
24
|
+
* connection wait has already been given up, so the group reports `timedOut` instead of
|
|
25
|
+
* `progressing` forever — the reading that lets one held websocket outlive the activation
|
|
26
|
+
* budget and stall every later deploy. `timedOut` names the deadline; it is not an escalation
|
|
27
|
+
* and never authorises a signal.
|
|
28
|
+
* @param {object} args - Classification inputs.
|
|
29
|
+
* @param {number} args.connectionCount - Live proxy connections this generation still owns.
|
|
30
|
+
* @param {DrainLivenessSample} args.currentSample - Newest liveness observation.
|
|
31
|
+
* @param {number} args.drainBudgetMs - Configured drain budget (`proxy.drainTimeoutMs`); 0 records no budget.
|
|
32
|
+
* @param {number} args.drainElapsedMs - Time since this drain started, at classification time.
|
|
33
|
+
* @param {number} args.minSampleWindowMs - Shortest window that can prove flat CPU.
|
|
34
|
+
* @param {DrainLivenessSample | undefined} args.previousSample - Older observation, when one is in range.
|
|
35
|
+
* @returns {DrainClassification} The classification and the evidence it rests on.
|
|
36
|
+
*/
|
|
37
|
+
export function classifyDrainProgress({connectionCount, currentSample, drainBudgetMs = 0, drainElapsedMs = 0, minSampleWindowMs, previousSample}) {
|
|
38
|
+
const live = currentSample.processes.filter((process) => process.pid !== undefined || process.heldProcessGroupPgid !== undefined)
|
|
39
|
+
const stillInsideBudget = live.filter((process) => !process.retirementTimedOut)
|
|
40
|
+
|
|
41
|
+
if (connectionCount > 0) {
|
|
42
|
+
const connections = `${connectionCount} live connection${connectionCount === 1 ? "" : "s"} still owned by the generation`
|
|
43
|
+
|
|
44
|
+
if (drainBudgetMs > 0 && drainElapsedMs > drainBudgetMs) {
|
|
45
|
+
return classify("timedOut", `drain budget ${drainBudgetMs}ms exceeded by ${drainElapsedMs - drainBudgetMs}ms with ${connections}`, currentSample, previousSample)
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
return classify("progressing", connections, currentSample, previousSample)
|
|
49
|
+
}
|
|
50
|
+
if (stillInsideBudget.length > 0) {
|
|
51
|
+
return classify("progressing", `process ${stillInsideBudget[0].id} is still inside its retirement budget`, currentSample, previousSample)
|
|
52
|
+
}
|
|
53
|
+
if (live.length === 0) {
|
|
54
|
+
return classify("progressing", "no live process remains in the generation", currentSample, previousSample)
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
const liveChildCount = live.reduce((total, process) => total + (process.liveChildCount ?? 0), 0)
|
|
58
|
+
|
|
59
|
+
if (liveChildCount === 0) {
|
|
60
|
+
return classify("wedged", `${live.length} live process${live.length === 1 ? "" : "es"} past its retirement budget with no runner child remaining${heldGroupNote(live)}`, currentSample, previousSample)
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
if (!previousSample) {
|
|
64
|
+
return classify("unproven", `first liveness observation; a ${minSampleWindowMs}ms window is required before CPU flatness can be claimed`, currentSample, previousSample)
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const sampleWindowMs = currentSample.atMs - previousSample.atMs
|
|
68
|
+
|
|
69
|
+
if (sampleWindowMs < minSampleWindowMs) {
|
|
70
|
+
return classify("unproven", `liveness window ${sampleWindowMs}ms is below the ${minSampleWindowMs}ms minimum`, currentSample, previousSample)
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const currentCpuMs = totalCpuMs(currentSample)
|
|
74
|
+
const previousCpuMs = totalCpuMs(previousSample)
|
|
75
|
+
|
|
76
|
+
if (currentCpuMs === undefined || previousCpuMs === undefined) {
|
|
77
|
+
return classify("unproven", "process CPU is not measurable on this host", currentSample, previousSample)
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
const cpuDeltaMs = currentCpuMs - previousCpuMs
|
|
81
|
+
|
|
82
|
+
if (cpuDeltaMs !== 0) {
|
|
83
|
+
const change = cpuDeltaMs > 0
|
|
84
|
+
? `CPU advanced ${cpuDeltaMs}ms`
|
|
85
|
+
: `the group changed (${live.length} live process${live.length === 1 ? "" : "es"}, ${cpuDeltaMs}ms of consumed CPU left the group)`
|
|
86
|
+
|
|
87
|
+
return classify("progressing", `${change} across ${sampleWindowMs}ms`, currentSample, previousSample)
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
return classify("wedged", `${live.length} live process${live.length === 1 ? "" : "es"} with ${liveChildCount} runner ${liveChildCount === 1 ? "child" : "children"}, no live connection, every retirement budget expired, CPU flat across ${sampleWindowMs}ms${heldGroupNote(live)}`, currentSample, previousSample)
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Decides whether a new release generation may be activated under the draining-group bound.
|
|
95
|
+
*
|
|
96
|
+
* The incumbent counts as a draining group because activating the candidate retires it: the
|
|
97
|
+
* count is what the fleet would have *after* the handoff, which is the only version of the
|
|
98
|
+
* number a pre-handoff decision can act on.
|
|
99
|
+
* @param {object} args - Admission inputs.
|
|
100
|
+
* @param {string} args.candidateReleaseId - Release the handoff would activate.
|
|
101
|
+
* @param {DrainGroupReport[]} args.draining - Every release group currently draining.
|
|
102
|
+
* @param {boolean} args.incumbentWillDrain - Whether an active generation is retired by this handoff.
|
|
103
|
+
* @param {number} args.maxConcurrentDrainingReleaseGroups - Policy bound; 0 disables the check.
|
|
104
|
+
* @returns {GenerationAdmission} Allow/deny plus the named reason and the groups behind it.
|
|
105
|
+
*/
|
|
106
|
+
export function admitGenerationRelease({candidateReleaseId, draining, incumbentWillDrain, maxConcurrentDrainingReleaseGroups}) {
|
|
107
|
+
const wouldBeDrainingCount = draining.length + (incumbentWillDrain ? 1 : 0)
|
|
108
|
+
|
|
109
|
+
if (maxConcurrentDrainingReleaseGroups === 0 || wouldBeDrainingCount <= maxConcurrentDrainingReleaseGroups) {
|
|
110
|
+
return {allowed: true, draining, reason: undefined, wouldBeDrainingCount}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
const reported = draining.map((group) => `${group.releaseId} ${group.status} (${group.reason})`).join("; ")
|
|
114
|
+
const reason = `Deploy refused before activation: ${draining.length} release group${draining.length === 1 ? "" : "s"} already draining${reported ? ` (${reported})` : ""}. Activating ${candidateReleaseId} would make ${wouldBeDrainingCount}, above the releaseRetention.maxConcurrentDrainingReleaseGroups bound of ${maxConcurrentDrainingReleaseGroups}.`
|
|
115
|
+
|
|
116
|
+
return {allowed: false, draining, reason, wouldBeDrainingCount}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* Decides whether a proven wedge may be terminated by the generation supervisor.
|
|
121
|
+
*
|
|
122
|
+
* A wedge claim is not a kill licence. The claim says a drain cannot progress any more; a
|
|
123
|
+
* SIGKILL is only safe when it also cannot destroy work. That needs four separate proofs: the
|
|
124
|
+
* generation is classified wedged (so it owns no live connection), every live process is
|
|
125
|
+
* already past its retirement budget, no runner child remains in any tracked process group —
|
|
126
|
+
* STOE-1077 measured 67 builds and 27.3 build-hours lost to exactly this signal reaching live
|
|
127
|
+
* lanes — and the group's CPU is proven flat across a full sample window, which rules out a
|
|
128
|
+
* group that is merely slow. Anything else reports, with the missing proof named.
|
|
129
|
+
* @param {object} args - Decision inputs.
|
|
130
|
+
* @param {DrainClassification} args.classification - The classification the wedge claim rests on.
|
|
131
|
+
* @param {DrainLivenessSample} args.currentSample - Newest liveness observation, the one the decision acts on.
|
|
132
|
+
* @param {number} args.minSampleWindowMs - Shortest window that can prove flat CPU.
|
|
133
|
+
* @returns {EscalationDecision} Escalate with named targets, or report the missing proof.
|
|
134
|
+
*/
|
|
135
|
+
export function decideWedgedEscalation({classification, currentSample, minSampleWindowMs}) {
|
|
136
|
+
if (classification.status !== "wedged") {
|
|
137
|
+
return reportEscalation(`no escalation: drain is ${classification.status} (${classification.reason})`)
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
const live = currentSample.processes.filter((process) => process.pid !== undefined || process.heldProcessGroupPgid !== undefined)
|
|
141
|
+
|
|
142
|
+
if (live.length === 0) {
|
|
143
|
+
return reportEscalation("no escalation: the generation holds no live process")
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
const insideBudget = live.filter((process) => !process.retirementTimedOut)
|
|
147
|
+
|
|
148
|
+
if (insideBudget.length > 0) {
|
|
149
|
+
return reportEscalation(`no escalation: process ${insideBudget[0].id} has not expired its retirement budget`)
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
const liveChildCount = live.reduce((total, process) => total + (process.liveChildCount ?? 0), 0)
|
|
153
|
+
|
|
154
|
+
if (liveChildCount > 0) {
|
|
155
|
+
return reportEscalation(`no escalation: ${liveChildCount} runner ${liveChildCount === 1 ? "child is" : "children are"} still live in the generation; SIGKILL would abort in-flight work`)
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
if (classification.sampleWindowMs < minSampleWindowMs) {
|
|
159
|
+
return reportEscalation(`no escalation: liveness window ${classification.sampleWindowMs}ms is below the ${minSampleWindowMs}ms minimum`)
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
if (classification.cpuDeltaMs !== 0) {
|
|
163
|
+
const evidence = classification.cpuDeltaMs === undefined
|
|
164
|
+
? "process CPU is not measurable on this host"
|
|
165
|
+
: `${classification.cpuDeltaMs}ms of CPU moved across ${classification.sampleWindowMs}ms`
|
|
166
|
+
|
|
167
|
+
return reportEscalation(`no escalation: CPU flatness is not proven (${evidence})`)
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
const targets = live.flatMap((process) => {
|
|
171
|
+
const pgid = process.heldProcessGroupPgid ?? process.pid
|
|
172
|
+
|
|
173
|
+
return pgid === undefined ? [] : [{pgid, processId: process.id}]
|
|
174
|
+
})
|
|
175
|
+
|
|
176
|
+
if (targets.length === 0) {
|
|
177
|
+
return reportEscalation("no escalation: no process group id is addressable")
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
return {
|
|
181
|
+
decision: "escalate",
|
|
182
|
+
killSafe: true,
|
|
183
|
+
reason: `escalating: ${live.length} live process${live.length === 1 ? "" : "es"} past its retirement budget, no runner child remaining, CPU flat across ${classification.sampleWindowMs}ms (${classification.reason})`,
|
|
184
|
+
targets
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* @param {string} reason - The proof the escalation is missing.
|
|
190
|
+
* @returns {EscalationDecision} Report-only decision.
|
|
191
|
+
*/
|
|
192
|
+
function reportEscalation(reason) {
|
|
193
|
+
return {decision: "report", killSafe: false, reason, targets: []}
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/**
|
|
197
|
+
* Names the process groups that outlived their tracked process.
|
|
198
|
+
*
|
|
199
|
+
* A tracked process can exit while the group it led keeps running (a shell wrapper killed by
|
|
200
|
+
* the stop signal, a runner child that outlives its parent). Those members still hold the
|
|
201
|
+
* generation's work and no rollbridge record can signal them, so a wedge claim has to say
|
|
202
|
+
* which group is holding it.
|
|
203
|
+
* @param {DrainProcessObservation[]} processes - Live observations of one generation.
|
|
204
|
+
* @returns {string} A note naming the held groups, or "" when every group is still tracked.
|
|
205
|
+
*/
|
|
206
|
+
function heldGroupNote(processes) {
|
|
207
|
+
const held = processes.filter((process) => process.heldProcessGroupPgid !== undefined)
|
|
208
|
+
|
|
209
|
+
if (held.length === 0) return ""
|
|
210
|
+
|
|
211
|
+
const named = held.map((process) => `${process.id} pgid ${process.heldProcessGroupPgid}`).join(", ")
|
|
212
|
+
|
|
213
|
+
return `; ${held.length} tracked process${held.length === 1 ? "" : "es"} already exited and left ${named} alive`
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/**
|
|
217
|
+
* @param {DrainProgress} status - Classification.
|
|
218
|
+
* @param {string} reason - Named evidence behind it.
|
|
219
|
+
* @param {DrainLivenessSample} currentSample - Newest observation.
|
|
220
|
+
* @param {DrainLivenessSample | undefined} previousSample - Older observation, when in range.
|
|
221
|
+
* @returns {DrainClassification} Classification payload.
|
|
222
|
+
*/
|
|
223
|
+
function classify(status, reason, currentSample, previousSample) {
|
|
224
|
+
const sampleWindowMs = previousSample ? currentSample.atMs - previousSample.atMs : 0
|
|
225
|
+
const currentCpuMs = totalCpuMs(currentSample)
|
|
226
|
+
const previousCpuMs = totalCpuMs(previousSample)
|
|
227
|
+
|
|
228
|
+
return {
|
|
229
|
+
cpuDeltaMs: currentCpuMs === undefined || previousCpuMs === undefined ? undefined : currentCpuMs - previousCpuMs,
|
|
230
|
+
reason,
|
|
231
|
+
sampleWindowMs,
|
|
232
|
+
status
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
/**
|
|
237
|
+
* @param {DrainLivenessSample | undefined} sample - Observation to total.
|
|
238
|
+
* @returns {number | undefined} Summed CPU, or undefined when any live process is unmeasurable.
|
|
239
|
+
*/
|
|
240
|
+
function totalCpuMs(sample) {
|
|
241
|
+
if (!sample) return undefined
|
|
242
|
+
|
|
243
|
+
let total = 0
|
|
244
|
+
|
|
245
|
+
for (const process of sample.processes) {
|
|
246
|
+
if (process.pid === undefined && process.heldProcessGroupPgid === undefined) continue
|
|
247
|
+
if (process.cpuMs === undefined) return undefined
|
|
248
|
+
total += process.cpuMs
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
return total
|
|
252
|
+
}
|
package/src/guardian-client.js
CHANGED
|
@@ -2,17 +2,34 @@
|
|
|
2
2
|
|
|
3
3
|
import crypto from "node:crypto"
|
|
4
4
|
import {spawn} from "node:child_process"
|
|
5
|
+
import fs from "node:fs/promises"
|
|
5
6
|
import net from "node:net"
|
|
7
|
+
import path from "node:path"
|
|
6
8
|
import {fileURLToPath} from "node:url"
|
|
7
9
|
import ManagedProcess from "./managed-process.js"
|
|
8
10
|
|
|
9
11
|
const guardianPath = fileURLToPath(new URL("./process-guardian.js", import.meta.url))
|
|
12
|
+
const MAX_UNIX_SOCKET_PATH_BYTES = 107
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Linux reserves one byte of sockaddr_un.sun_path for its terminator. Keep
|
|
16
|
+
* long task/workspace prefixes out of the pathname while retaining a stable,
|
|
17
|
+
* collision-resistant name for the configured path.
|
|
18
|
+
* @param {string} socketPath - Configured Unix socket pathname.
|
|
19
|
+
* @returns {string} Path that can be passed to net.createConnection/listen.
|
|
20
|
+
*/
|
|
21
|
+
export function usableSocketPath(socketPath) {
|
|
22
|
+
if (Buffer.byteLength(socketPath) <= MAX_UNIX_SOCKET_PATH_BYTES) return socketPath
|
|
23
|
+
|
|
24
|
+
const digest = crypto.createHash("sha256").update(socketPath).digest("hex").slice(0, 32)
|
|
25
|
+
return path.join("/tmp", `rollbridge-${digest}.sock`)
|
|
26
|
+
}
|
|
10
27
|
|
|
11
28
|
export default class GuardianClient {
|
|
12
29
|
/** @param {{pid?: number, socketPath: string, token: string}} identity - Durable guardian identity. */
|
|
13
30
|
constructor({pid, socketPath, token}) {
|
|
14
31
|
this.pid = pid
|
|
15
|
-
this.socketPath = socketPath
|
|
32
|
+
this.socketPath = usableSocketPath(socketPath)
|
|
16
33
|
this.token = token
|
|
17
34
|
this.socket = /** @type {net.Socket | undefined} */ (undefined)
|
|
18
35
|
this.buffer = ""
|
|
@@ -150,14 +167,17 @@ export default class GuardianClient {
|
|
|
150
167
|
return await response
|
|
151
168
|
}
|
|
152
169
|
|
|
153
|
-
/**
|
|
154
|
-
|
|
170
|
+
/**
|
|
171
|
+
* Stops the guardian after every owned process has stopped or reached its retirement budget.
|
|
172
|
+
* @param {{retirementTimeoutMs?: number}} [options] - Optional non-blocking retirement budget.
|
|
173
|
+
*/
|
|
174
|
+
async shutdown(options = {}) {
|
|
155
175
|
if (this.pending.size > 0) {
|
|
156
176
|
await new Promise((resolve) => {
|
|
157
177
|
this.idleWaiters.push(() => { resolve(undefined) })
|
|
158
178
|
})
|
|
159
179
|
}
|
|
160
|
-
await this.request({command: "shutdown"})
|
|
180
|
+
await this.request({command: "shutdown", options})
|
|
161
181
|
const socket = this.socket
|
|
162
182
|
|
|
163
183
|
if (!socket || socket.destroyed) throw new Error("Process guardian disconnected before shutdown acknowledgement")
|
|
@@ -186,9 +206,11 @@ export default class GuardianClient {
|
|
|
186
206
|
await this.request({command: "owner-ready", ownerPid: process.pid})
|
|
187
207
|
}
|
|
188
208
|
|
|
189
|
-
/** @returns {Promise<{daemonRecovery: number, generationReactivation?: number}>} Guardian protocol capabilities. */
|
|
209
|
+
/** @returns {Promise<{daemonRecovery: number, generationReactivation?: number, guardianVersion?: string}>} Guardian protocol capabilities. */
|
|
190
210
|
async capabilities() {
|
|
191
|
-
const capabilities = /** @type {{daemonRecovery: number, generationReactivation?: number}} */ (await this.request({command: "capabilities"}))
|
|
211
|
+
const capabilities = /** @type {{daemonRecovery: number, generationReactivation?: number, guardianVersion?: string}} */ (await this.request({command: "capabilities"}))
|
|
212
|
+
|
|
213
|
+
if (!capabilities.guardianVersion) capabilities.guardianVersion = await guardianVersionFromProcess(this.pid)
|
|
192
214
|
|
|
193
215
|
this.generationReactivation = capabilities.generationReactivation === 1
|
|
194
216
|
return capabilities
|
|
@@ -323,10 +345,11 @@ export default class GuardianClient {
|
|
|
323
345
|
/**
|
|
324
346
|
* Acquires the committed owner's mutation fence.
|
|
325
347
|
* @param {string} operation - Control mutation name.
|
|
348
|
+
* @param {{mutationTimeoutMs?: number, ownerPid?: number, waitingFor?: string}} [details] - Lease bound, holding PID and what the mutation blocks on.
|
|
326
349
|
* @returns {Promise<string>} Mutation lease id.
|
|
327
350
|
*/
|
|
328
|
-
async beginOwnerMutation(operation) {
|
|
329
|
-
const result = /** @type {{mutationId: string}} */ (await this.request({command: "begin-owner-mutation", operation}))
|
|
351
|
+
async beginOwnerMutation(operation, {mutationTimeoutMs, ownerPid, waitingFor} = {}) {
|
|
352
|
+
const result = /** @type {{mutationId: string}} */ (await this.request({command: "begin-owner-mutation", mutationTimeoutMs, operation, ownerPid, waitingFor}))
|
|
330
353
|
|
|
331
354
|
return result.mutationId
|
|
332
355
|
}
|
|
@@ -336,6 +359,21 @@ export default class GuardianClient {
|
|
|
336
359
|
await this.request({command: "end-owner-mutation", mutationId})
|
|
337
360
|
}
|
|
338
361
|
|
|
362
|
+
/**
|
|
363
|
+
* Refines what a held mutation is waiting for, so a later refusal names the real blocker.
|
|
364
|
+
* @param {string} mutationId - Mutation lease id.
|
|
365
|
+
* @param {string} waitingFor - Current blocker.
|
|
366
|
+
* @returns {Promise<void>} Report acknowledged.
|
|
367
|
+
*/
|
|
368
|
+
async reportOwnerMutationWait(mutationId, waitingFor) {
|
|
369
|
+
await this.request({command: "report-owner-mutation-wait", mutationId, waitingFor})
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
/** @returns {Promise<{guardianPid: number, ownerMutation: {elapsedMs: number, leaseMs: number, mutationId: string, operation: string, pid: number | null, startedAt: string, waitingFor: string} | null, version: string}>} Guardian diagnostics. */
|
|
373
|
+
async guardianStatus() {
|
|
374
|
+
return /** @type {{guardianPid: number, ownerMutation: {elapsedMs: number, leaseMs: number, mutationId: string, operation: string, pid: number | null, startedAt: string, waitingFor: string} | null, version: string}} */ (await this.request({command: "guardian-status"}))
|
|
375
|
+
}
|
|
376
|
+
|
|
339
377
|
/** @returns {Promise<{committedReplacementId: string | null, ownerClaimed: boolean, retirementFailed?: boolean, retirementPending?: boolean, retirementReady?: boolean}>} Transaction status. */
|
|
340
378
|
async replacementStatus() {
|
|
341
379
|
return /** @type {{committedReplacementId: string | null, ownerClaimed: boolean, retirementFailed?: boolean, retirementPending?: boolean, retirementReady?: boolean}} */ (await this.request({command: "replacement-status"}))
|
|
@@ -439,6 +477,27 @@ export default class GuardianClient {
|
|
|
439
477
|
}
|
|
440
478
|
}
|
|
441
479
|
|
|
480
|
+
/**
|
|
481
|
+
* Reads the package version from a guardian's own executable path. Used when its
|
|
482
|
+
* capabilities response predates guardian version reporting, and by owner
|
|
483
|
+
* replacement, which must not send a capabilities command before a replacement
|
|
484
|
+
* transaction is prepared. An unreadable or non-Rollbridge process remains unknown.
|
|
485
|
+
* @param {number | undefined} pid - Existing guardian PID.
|
|
486
|
+
* @returns {Promise<string | undefined>} Guardian package version.
|
|
487
|
+
*/
|
|
488
|
+
export async function guardianVersionFromProcess(pid) {
|
|
489
|
+
if (!pid) return undefined
|
|
490
|
+
|
|
491
|
+
try {
|
|
492
|
+
const commandLine = (await fs.readFile(`/proc/${pid}/cmdline`, "utf8")).split("\0").find((argument) => argument.endsWith("/src/process-guardian.js"))
|
|
493
|
+
if (!commandLine) return undefined
|
|
494
|
+
const metadata = JSON.parse(await fs.readFile(path.resolve(path.dirname(commandLine), "..", "package.json"), "utf8"))
|
|
495
|
+
return metadata && typeof metadata.version === "string" ? metadata.version : undefined
|
|
496
|
+
} catch {
|
|
497
|
+
return undefined
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
|
|
442
501
|
class GuardianProcess extends ManagedProcess {
|
|
443
502
|
/** @param {{client: GuardianClient, definition: ConstructorParameters<typeof ManagedProcess>[0], key: string}} args - Remote process args. */
|
|
444
503
|
constructor({client, definition, key}) {
|
|
@@ -457,8 +516,32 @@ class GuardianProcess extends ManagedProcess {
|
|
|
457
516
|
if (!this.registration) {
|
|
458
517
|
this.registration = this.client.request({command: "register", definition: this.definition, key: this.key, provenance: this.provenance})
|
|
459
518
|
.then((status) => { this.cachedStatus = asProcessStatus(status) })
|
|
519
|
+
.catch(async (error) => {
|
|
520
|
+
// Guardians predating stale-record re-registration reject a restore before
|
|
521
|
+
// the current guardian can apply its stopped/failed-record fence. Reconcile
|
|
522
|
+
// only an exact, conclusively dead record through the authenticated remove
|
|
523
|
+
// command; live or ambiguous records remain fail-closed on provenance.
|
|
524
|
+
if (!(error instanceof Error) || !error.message.startsWith(`Guardian provenance mismatch for ${this.key}`)) throw error
|
|
525
|
+
const stale = (await this.client.inventory()).find((entry) => entry.key === this.key)
|
|
526
|
+
const staleStatus = stale?.status
|
|
527
|
+
const staleIsGone = staleStatus &&
|
|
528
|
+
(staleStatus.state === "stopped" || staleStatus.state === "failed") &&
|
|
529
|
+
staleStatus.pid == null &&
|
|
530
|
+
(!staleStatus.heldProcessGroup || staleStatus.heldProcessGroup.liveMemberCount === 0)
|
|
531
|
+
|
|
532
|
+
if (!stale || !staleIsGone) throw error
|
|
533
|
+
await this.client.remove(stale.key, stale.provenance)
|
|
534
|
+
const status = await this.client.request({command: "register", definition: this.definition, key: this.key, provenance: this.provenance})
|
|
535
|
+
|
|
536
|
+
this.cachedStatus = asProcessStatus(status)
|
|
537
|
+
})
|
|
538
|
+
}
|
|
539
|
+
try {
|
|
540
|
+
await this.registration
|
|
541
|
+
} catch (error) {
|
|
542
|
+
this.registration = undefined
|
|
543
|
+
throw error
|
|
460
544
|
}
|
|
461
|
-
await this.registration
|
|
462
545
|
}
|
|
463
546
|
|
|
464
547
|
/** Reconnects to an already registered guardian process without changing its desired state. */
|
package/src/health.js
CHANGED
|
@@ -1,35 +1,126 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
|
+
import net from "node:net"
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* @typedef {import("./config.js").HealthConfig} HealthConfig
|
|
7
|
+
* @typedef {{attempts: number, deadlineMs: number, elapsedMs: number, processId: string, url: string}} HealthCheckDiagnostics
|
|
8
|
+
* @typedef {Error & {healthCheck?: HealthCheckDiagnostics}} HealthCheckError
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* @param {number} ms - Milliseconds.
|
|
13
|
+
* @returns {Promise<void>} Resolves after the delay.
|
|
14
|
+
*/
|
|
15
|
+
function sleep(ms) {
|
|
16
|
+
return new Promise((resolve) => setTimeout(resolve, ms))
|
|
17
|
+
}
|
|
18
|
+
|
|
3
19
|
/**
|
|
4
20
|
* Waits for a process HTTP health endpoint.
|
|
21
|
+
*
|
|
22
|
+
* The failure names its own budget: `Health check failed ...: fetch failed` cannot tell a
|
|
23
|
+
* slow boot apart from a health deadline that was simply too short.
|
|
5
24
|
* @param {object} args - Options.
|
|
6
25
|
* @param {string} args.host - Host.
|
|
7
26
|
* @param {number} args.port - Port.
|
|
8
|
-
* @param {
|
|
9
|
-
* @
|
|
27
|
+
* @param {HealthConfig} args.health - Health config.
|
|
28
|
+
* @param {string} [args.processId] - Process the probe belongs to, named in the failure.
|
|
29
|
+
* @returns {Promise<{elapsedMs: number}>} Time spent reaching the first healthy response.
|
|
10
30
|
*/
|
|
11
|
-
export async function waitForHealth({health, host, port}) {
|
|
31
|
+
export async function waitForHealth({health, host, port, processId = "proxied"}) {
|
|
32
|
+
const startedAtMs = Date.now()
|
|
33
|
+
|
|
12
34
|
if (health.startDelayMs > 0) {
|
|
13
|
-
await
|
|
35
|
+
await sleep(health.startDelayMs)
|
|
14
36
|
}
|
|
15
37
|
|
|
16
|
-
const
|
|
38
|
+
const deadlineMs = health.timeoutMs
|
|
39
|
+
const deadline = Date.now() + deadlineMs
|
|
17
40
|
const url = `http://${host}:${port}${health.path}`
|
|
41
|
+
let attempts = 0
|
|
18
42
|
let lastError = "no attempts"
|
|
19
43
|
|
|
20
44
|
while (Date.now() <= deadline) {
|
|
45
|
+
attempts += 1
|
|
46
|
+
|
|
21
47
|
try {
|
|
22
48
|
const response = await fetch(url)
|
|
23
49
|
|
|
24
|
-
if (response.ok) return
|
|
50
|
+
if (response.ok) return {elapsedMs: Date.now() - startedAtMs}
|
|
25
51
|
|
|
26
52
|
lastError = `HTTP ${response.status}`
|
|
27
53
|
} catch (error) {
|
|
28
54
|
lastError = error instanceof Error ? error.message : String(error)
|
|
29
55
|
}
|
|
30
56
|
|
|
31
|
-
await
|
|
57
|
+
await sleep(health.intervalMs)
|
|
32
58
|
}
|
|
33
59
|
|
|
34
|
-
|
|
60
|
+
const elapsedMs = Date.now() - startedAtMs
|
|
61
|
+
/** @type {HealthCheckError} */
|
|
62
|
+
const failure = new Error(`Health check failed for ${url}: ${lastError} (process ${processId}: ${attempts} probes, ${elapsedMs}ms elapsed of the ${deadlineMs}ms health deadline)`)
|
|
63
|
+
|
|
64
|
+
failure.healthCheck = {attempts, deadlineMs, elapsedMs, processId, url}
|
|
65
|
+
|
|
66
|
+
throw failure
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Waits until a TCP endpoint accepts a connection, on a budget of its own.
|
|
71
|
+
*
|
|
72
|
+
* This is the dependency leg of a release start: waiting for a process that must bind a
|
|
73
|
+
* port before another one can boot. Charged to the health deadline it measures the
|
|
74
|
+
* dependency instead of the app, so it is timed here on a separate budget.
|
|
75
|
+
* @param {object} args - Options.
|
|
76
|
+
* @param {string} args.host - Host to connect to.
|
|
77
|
+
* @param {number} args.intervalMs - Delay between connection attempts.
|
|
78
|
+
* @param {number} args.port - Port that must accept a connection.
|
|
79
|
+
* @param {number} args.timeoutMs - Budget for the wait.
|
|
80
|
+
* @returns {Promise<{attempts: number, elapsedMs: number}>} Time and attempts spent waiting.
|
|
81
|
+
*/
|
|
82
|
+
export async function waitForListeningPort({host, intervalMs, port, timeoutMs}) {
|
|
83
|
+
const startedAtMs = Date.now()
|
|
84
|
+
const deadline = startedAtMs + timeoutMs
|
|
85
|
+
let attempts = 0
|
|
86
|
+
|
|
87
|
+
while (Date.now() <= deadline) {
|
|
88
|
+
attempts += 1
|
|
89
|
+
|
|
90
|
+
if (await tryConnect({deadline, host, intervalMs, port})) {
|
|
91
|
+
return {attempts, elapsedMs: Date.now() - startedAtMs}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
await sleep(Math.max(1, Math.min(intervalMs, deadline - Date.now())))
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
throw new Error(`Dependency endpoint ${host}:${port} did not accept a connection within ${timeoutMs}ms (${attempts} attempts, ${Date.now() - startedAtMs}ms waited)`)
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* @param {object} args - Attempt options.
|
|
102
|
+
* @param {number} args.deadline - Absolute deadline that bounds this attempt.
|
|
103
|
+
* @param {string} args.host - Host to connect to.
|
|
104
|
+
* @param {number} args.intervalMs - Delay between connection attempts.
|
|
105
|
+
* @param {number} args.port - Port to connect to.
|
|
106
|
+
* @returns {Promise<boolean>} True when the endpoint accepted the connection.
|
|
107
|
+
*/
|
|
108
|
+
function tryConnect({deadline, host, intervalMs, port}) {
|
|
109
|
+
return new Promise((resolve) => {
|
|
110
|
+
const socket = net.createConnection({host, port})
|
|
111
|
+
let settled = false
|
|
112
|
+
|
|
113
|
+
/** @param {boolean} connected - Whether the endpoint accepted the connection. */
|
|
114
|
+
const finish = (connected) => {
|
|
115
|
+
if (settled) return
|
|
116
|
+
settled = true
|
|
117
|
+
socket.destroy()
|
|
118
|
+
resolve(connected)
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
socket.setTimeout(Math.max(1, Math.min(intervalMs, deadline - Date.now())))
|
|
122
|
+
socket.once("connect", () => finish(true))
|
|
123
|
+
socket.once("error", () => finish(false))
|
|
124
|
+
socket.once("timeout", () => finish(false))
|
|
125
|
+
})
|
|
35
126
|
}
|