rollbridge 0.1.68 → 0.1.70
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -9
- package/TODO.md +1 -0
- package/changelog.d/20261116-guardian-version-handoff.md +37 -0
- package/changelog.d/20261116-reclaim-dead-owner-mutation.md +7 -0
- package/changelog.d/20261117-activate-before-incumbent-retirement.md +1 -0
- package/changelog.d/20261117-activation-path-drain-deadline.md +35 -0
- package/changelog.d/20261117-stop-companion-after-retired-dependency.md +7 -0
- package/docs/cli.md +14 -7
- package/docs/config.md +8 -4
- package/docs/troubleshooting.md +27 -0
- package/docs/workers.md +1 -0
- package/examples/tensorbuzz.com.js +2 -1
- package/package.json +1 -1
- package/src/daemon.js +178 -48
- package/src/doctor.js +7 -0
- package/src/drain-policy.js +18 -4
- package/src/guardian-client.js +54 -6
- package/src/managed-process.js +14 -5
- package/src/process-guardian.js +76 -5
- package/src/release-group.js +40 -4
- package/tensorbuzz.yml +3 -1
- package/test/drain-policy.test.js +28 -0
- package/test/guardian-client.test.js +49 -5
- package/test/managed-process.test.js +3 -1
- package/test/owner-recovery.test.js +52 -13
- package/test/owner-replacement.test.js +97 -12
- package/test/release-group.test.js +57 -0
- package/test/rollbridge.test.js +84 -28
- package/test/shutdown-completion.test.js +52 -0
package/src/managed-process.js
CHANGED
|
@@ -22,7 +22,7 @@ const ESCALATION_KILL_CONFIRM_MS = 5000
|
|
|
22
22
|
* @typedef {import("node:child_process").ChildProcess["signalCode"]} ProcessExitSignal
|
|
23
23
|
* @typedef {{at: string, line: string, stream: "stdout" | "stderr"}} ManagedProcessLog
|
|
24
24
|
* @typedef {import("./config.js").StopTimeoutMs} StopTimeoutMs
|
|
25
|
-
* @typedef {{command: string, cwd: string | undefined, env: Record<string, string | undefined>, lifecycle: import("./config.js").LifecycleConfig, logger: (message: string, data?: Record<string, import("./json.js").JsonValue>) => void, memory: import("./config.js").MemoryConfig | undefined, outputLines: number, restart: import("./config.js").RestartConfig, restartDelayMs: number, shouldRestart: () => boolean, stopSignal: string, stopTimeoutMs: StopTimeoutMs}} ManagedProcessDefinition
|
|
25
|
+
* @typedef {{command: string, cwd: string | undefined, env: Record<string, string | undefined>, lifecycle: import("./config.js").LifecycleConfig, logger: (message: string, data?: Record<string, import("./json.js").JsonValue>) => void, memory: import("./config.js").MemoryConfig | undefined, outputLines: number, restart: import("./config.js").RestartConfig, restartDelayMs: number, shouldRestart: () => boolean, shouldStop?: () => boolean, stopSignal: string, stopTimeoutMs: StopTimeoutMs}} ManagedProcessDefinition
|
|
26
26
|
* @typedef {{liveMemberCount: number | undefined, pgid: number}} HeldProcessGroup
|
|
27
27
|
* @typedef {{children: import("./process-memory.js").ProcessGroupMember[], command: string, cwd: string | undefined, exitCode: number | null | undefined, exitSignal: ProcessExitSignal | undefined, heldProcessGroup: HeldProcessGroup | undefined, id: string, lastMemoryRestartAt: string | undefined, lastStartReason: ManagedProcessStartReason | undefined, lifecycleRole?: LifecycleRole, logs: ManagedProcessLog[], memoryRestarts: number, pid: number | undefined, restarts: number, retirementTimedOut: boolean, rssBytes: number | undefined, startedAt: string | undefined, state: ManagedProcessState, uptimeMs: number | undefined}} ManagedProcessStatus
|
|
28
28
|
*/
|
|
@@ -41,10 +41,11 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
41
41
|
* @param {import("./config.js").RestartConfig} [args.restart] - Restart policy (defaults to unlimited restarts with a constant delay).
|
|
42
42
|
* @param {number} args.restartDelayMs - Restart delay.
|
|
43
43
|
* @param {() => boolean} args.shouldRestart - Restart policy callback.
|
|
44
|
+
* @param {() => boolean} [args.shouldStop] - Whether an unexpected exit is terminal because its dependency retired.
|
|
44
45
|
* @param {string} [args.stopSignal] - Signal sent to gracefully stop the process (default "SIGTERM").
|
|
45
46
|
* @param {StopTimeoutMs} args.stopTimeoutMs - Stop timeout.
|
|
46
47
|
*/
|
|
47
|
-
constructor({command, cwd, env, id, lifecycle = {drainTimeoutMs: 0}, logger, memory, outputLines, restart = {backoffFactor: 1, maxDelayMs: 0, maxRestarts: undefined, windowMs: 0}, restartDelayMs, shouldRestart, stopSignal = "SIGTERM", stopTimeoutMs}) {
|
|
48
|
+
constructor({command, cwd, env, id, lifecycle = {drainTimeoutMs: 0}, logger, memory, outputLines, restart = {backoffFactor: 1, maxDelayMs: 0, maxRestarts: undefined, windowMs: 0}, restartDelayMs, shouldRestart, shouldStop = () => false, stopSignal = "SIGTERM", stopTimeoutMs}) {
|
|
48
49
|
super()
|
|
49
50
|
|
|
50
51
|
this.command = command
|
|
@@ -58,6 +59,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
58
59
|
this.restart = restart
|
|
59
60
|
this.restartDelayMs = restartDelayMs
|
|
60
61
|
this.shouldRestart = shouldRestart
|
|
62
|
+
this.shouldStop = shouldStop
|
|
61
63
|
this.stopSignal = stopSignal
|
|
62
64
|
this.stopTimeoutMs = stopTimeoutMs
|
|
63
65
|
this.state = /** @type {ManagedProcessState} */ ("stopped")
|
|
@@ -207,6 +209,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
207
209
|
this.restart = definition.restart
|
|
208
210
|
this.restartDelayMs = definition.restartDelayMs
|
|
209
211
|
this.shouldRestart = definition.shouldRestart
|
|
212
|
+
this.shouldStop = definition.shouldStop ?? (() => false)
|
|
210
213
|
this.stopSignal = definition.stopSignal
|
|
211
214
|
this.stopTimeoutMs = definition.stopTimeoutMs
|
|
212
215
|
}
|
|
@@ -265,12 +268,13 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
265
268
|
this.rssBytes = undefined
|
|
266
269
|
this.children = []
|
|
267
270
|
this.clearMemoryMonitor()
|
|
268
|
-
|
|
271
|
+
const terminalExit = !wasIntentional && this.shouldStop()
|
|
272
|
+
this.state = wasIntentional || terminalExit ? "stopped" : "failed"
|
|
269
273
|
|
|
270
274
|
this.logger("process exited", {code, id: this.id, signal})
|
|
271
275
|
this.emit("exit", {code, signal})
|
|
272
276
|
|
|
273
|
-
if (!wasIntentional && this.shouldRestart()) {
|
|
277
|
+
if (!wasIntentional && !terminalExit && this.shouldRestart()) {
|
|
274
278
|
this.scheduleRestart()
|
|
275
279
|
}
|
|
276
280
|
}
|
|
@@ -507,6 +511,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
507
511
|
async performStop(options) {
|
|
508
512
|
const pgid = this.child?.pid ?? this.pid
|
|
509
513
|
const exitPromise = this.exitPromise
|
|
514
|
+
const retirementDeadline = options.retirementTimeoutMs === undefined ? undefined : Date.now() + options.retirementTimeoutMs
|
|
510
515
|
await this.quiesce()
|
|
511
516
|
|
|
512
517
|
if (!pgid) {
|
|
@@ -528,7 +533,6 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
528
533
|
// 3. Stop whatever is still running, then SIGKILL if it outlasts the graceful window.
|
|
529
534
|
if (this.processGroupExists(pgid)) {
|
|
530
535
|
const timeoutMs = options.timeoutMs ?? this.stopTimeoutMs
|
|
531
|
-
const retirementDeadline = options.retirementTimeoutMs === undefined ? undefined : Date.now() + options.retirementTimeoutMs
|
|
532
536
|
const gracefulDeadline = timeoutMs === "indefinite" ? undefined : Date.now() + timeoutMs
|
|
533
537
|
// A retirement budget is the caller's decision that this process must not be waited for and
|
|
534
538
|
// must not be killed, so it caps the graceful window rather than only standing in for a
|
|
@@ -550,6 +554,11 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
550
554
|
if (!exited && retirementBound) {
|
|
551
555
|
this.retirementTimedOut = true
|
|
552
556
|
this.markGroupOutlivedStop(pgid)
|
|
557
|
+
// The tracked process has completed its supervised stop even though its process group is
|
|
558
|
+
// still draining. Keep the residue in heldProcessGroup rather than leaving the process
|
|
559
|
+
// permanently quiesced, which would make a drained generation look active to restore
|
|
560
|
+
// admission and keep the owner state from describing the actual stopped generation.
|
|
561
|
+
this.state = "stopped"
|
|
553
562
|
this.logger("process retirement wait timed out; leaving process alive", {id: this.id, pid: pgid, timeoutMs: options.retirementTimeoutMs})
|
|
554
563
|
return
|
|
555
564
|
}
|
package/src/process-guardian.js
CHANGED
|
@@ -4,6 +4,7 @@ import fsSync from "node:fs"
|
|
|
4
4
|
import fs from "node:fs/promises"
|
|
5
5
|
import net from "node:net"
|
|
6
6
|
import crypto from "node:crypto"
|
|
7
|
+
import {createRequire} from "node:module"
|
|
7
8
|
import {spawn} from "node:child_process"
|
|
8
9
|
import {isDeepStrictEqual} from "node:util"
|
|
9
10
|
import path from "node:path"
|
|
@@ -12,6 +13,8 @@ import ManagedProcess, {childRecordIsGone} from "./managed-process.js"
|
|
|
12
13
|
import {writeState} from "./state-store.js"
|
|
13
14
|
|
|
14
15
|
const OWNER_RESTART_RETRY_MS = 1000
|
|
16
|
+
const commonJsRequire = createRequire(import.meta.url)
|
|
17
|
+
const GUARDIAN_VERSION = /** @type {{version: string}} */ (commonJsRequire("../package.json")).version
|
|
15
18
|
// An owner mutation is deploy/stop/restart/rollback/recovery work. A retained
|
|
16
19
|
// generation may legitimately drain for hours, but a mutation that holds the
|
|
17
20
|
// fence that long is stuck, so the lease is bounded and released on expiry.
|
|
@@ -28,7 +31,7 @@ const OWNER_MUTATION_LEASE_MS = 900000
|
|
|
28
31
|
* @property {number} id - Request id.
|
|
29
32
|
* @property {string} [key] - Stable process key.
|
|
30
33
|
* @property {boolean} [localSource] - Whether the retired daemon physically owns the published source.
|
|
31
|
-
* @property {{timeoutMs?: number}} [options] - Stop options.
|
|
34
|
+
* @property {{retirementTimeoutMs?: number, timeoutMs?: number}} [options] - Stop options.
|
|
32
35
|
* @property {string} [previousProvenance] - Expected current provenance.
|
|
33
36
|
* @property {string} [provenance] - New or registered provenance.
|
|
34
37
|
* @property {string} [releaseId] - Retained release identity.
|
|
@@ -173,7 +176,7 @@ const server = net.createServer((socket) => {
|
|
|
173
176
|
clearTimeout(waiter.timer)
|
|
174
177
|
waiter.reject(new Error("Owner claimant disconnected"))
|
|
175
178
|
}
|
|
176
|
-
if (ownerMutationClient === socket
|
|
179
|
+
if (ownerMutationClient === socket) clearOwnerMutationLease()
|
|
177
180
|
if (controlLessRetirement) {
|
|
178
181
|
const preCommitHandoff = replacementClient && ownerClient === socket && retiringReplacementId === replacementId
|
|
179
182
|
|
|
@@ -275,13 +278,14 @@ async function handleLine(socket, line) {
|
|
|
275
278
|
async function execute(request, socket) {
|
|
276
279
|
if (shuttingDown) throw new Error("Process guardian is shutting down")
|
|
277
280
|
|
|
278
|
-
if (request.command === "capabilities") return {daemonRecovery: 1, generationReactivation: 1}
|
|
281
|
+
if (request.command === "capabilities") return {daemonRecovery: 1, generationReactivation: 1, guardianVersion: GUARDIAN_VERSION}
|
|
279
282
|
|
|
280
283
|
if (request.command === "owner-replacement-capabilities") {
|
|
281
284
|
return {commands: ["commit-retired-owner-replacement"], protocol: "owner-replacement", version: 1}
|
|
282
285
|
}
|
|
283
286
|
|
|
284
287
|
if (request.command === "claim-owner") {
|
|
288
|
+
reclaimDeadOwnerMutation()
|
|
285
289
|
if (retirementFailed) throw new Error("Guardian owner replacement is fenced after incomplete retired-listener state transfer")
|
|
286
290
|
if (ownerState !== undefined && !isDeepStrictEqual(request.authority, ownerAuthority(ownerState))) {
|
|
287
291
|
const waitsForPreparedAuthority = Boolean(ownerClient && replacementOwnerState && isDeepStrictEqual(request.authority, replacementAuthority))
|
|
@@ -409,8 +413,11 @@ async function execute(request, socket) {
|
|
|
409
413
|
return {committedReplacementId: committedReplacementId ?? null, ownerClaimed: Boolean(ownerClient), retirementFailed, retirementPending: Boolean(retiringClient), retirementReady: retiringListenerReady}
|
|
410
414
|
}
|
|
411
415
|
|
|
416
|
+
if (request.command === "guardian-status") return guardianStatus()
|
|
417
|
+
|
|
412
418
|
if (request.command === "begin-owner-mutation") {
|
|
413
419
|
requireOwner(socket, request.command)
|
|
420
|
+
reclaimDeadOwnerMutation()
|
|
414
421
|
if (replacementClient) throw new Error(`Owner mutation ${request.operation || "operation"} is fenced while an owner replacement is prepared`)
|
|
415
422
|
const leaseMs = request.mutationTimeoutMs === undefined ? OWNER_MUTATION_LEASE_MS : request.mutationTimeoutMs
|
|
416
423
|
|
|
@@ -482,7 +489,7 @@ async function execute(request, socket) {
|
|
|
482
489
|
|
|
483
490
|
if (request.command === "stage-owner-replacement") {
|
|
484
491
|
requireReplacement(socket, request)
|
|
485
|
-
if (replacementRevision !== ownerRevision) throw new Error(
|
|
492
|
+
if (replacementRevision !== ownerRevision) throw new Error(`Guardian owner state changed after prepare (prepared revision ${replacementRevision}, current revision ${ownerRevision}); abort the stale candidate before preparing again`)
|
|
486
493
|
if (request.ownerState === undefined || !isDeepStrictEqual(ownerAuthority(request.ownerState), replacementAuthority)) {
|
|
487
494
|
throw new Error("Prepared replacement state does not match the requested authority")
|
|
488
495
|
}
|
|
@@ -613,7 +620,7 @@ async function execute(request, socket) {
|
|
|
613
620
|
requireOwner(socket, request.command)
|
|
614
621
|
shuttingDown = true
|
|
615
622
|
try {
|
|
616
|
-
const results = await Promise.allSettled([...processes.values()].map((entry) => entry.process.stop()))
|
|
623
|
+
const results = await Promise.allSettled([...processes.values()].map((entry) => entry.process.stop(request.options)))
|
|
617
624
|
const errors = results.filter((result) => result.status === "rejected").map((result) => result.reason)
|
|
618
625
|
|
|
619
626
|
if (errors.length > 0) throw new AggregateError(errors, `Guardian failed to stop ${errors.length} owned process${errors.length === 1 ? "" : "es"}: ${errors.map((error) => errorMessage(error instanceof Error ? error : String(error))).join("; ")}`)
|
|
@@ -772,6 +779,70 @@ function describeOwnerMutationLease() {
|
|
|
772
779
|
].join("; ")
|
|
773
780
|
}
|
|
774
781
|
|
|
782
|
+
/**
|
|
783
|
+
* Reports the currently held owner-mutation lease.
|
|
784
|
+
* @returns {{elapsedMs: number, leaseMs: number, mutationId: string, operation: string, pid: number | null, startedAt: string, waitingFor: string} | null} Active lease, if any.
|
|
785
|
+
*/
|
|
786
|
+
function activeOwnerMutationStatus() {
|
|
787
|
+
if (!ownerMutationClient || !ownerMutationId) return null
|
|
788
|
+
|
|
789
|
+
const startedAt = ownerMutationStartedAt ?? Date.now()
|
|
790
|
+
|
|
791
|
+
return {
|
|
792
|
+
elapsedMs: Math.max(0, Date.now() - startedAt),
|
|
793
|
+
leaseMs: ownerMutationLeaseMs,
|
|
794
|
+
mutationId: ownerMutationId,
|
|
795
|
+
operation: ownerMutationOperation || "unnamed operation",
|
|
796
|
+
pid: ownerMutationPid ?? ownerClientPid ?? null,
|
|
797
|
+
startedAt: new Date(startedAt).toISOString(),
|
|
798
|
+
waitingFor: ownerMutationWaitingFor || "the operation to report completion"
|
|
799
|
+
}
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
/**
|
|
803
|
+
* Reports guardian identity and owner-mutation diagnostics.
|
|
804
|
+
* @returns {{guardianPid: number, ownerMutation: ReturnType<typeof activeOwnerMutationStatus>, version: string}} Guardian diagnostics.
|
|
805
|
+
*/
|
|
806
|
+
function guardianStatus() {
|
|
807
|
+
return {guardianPid: process.pid, ownerMutation: activeOwnerMutationStatus(), version: GUARDIAN_VERSION}
|
|
808
|
+
}
|
|
809
|
+
|
|
810
|
+
/** @returns {boolean} Whether the recorded owner-mutation holder is no longer alive. */
|
|
811
|
+
function reclaimDeadOwnerMutation() {
|
|
812
|
+
if (!ownerMutationClient || !ownerMutationPid || isProcessAlive(ownerMutationPid)) return false
|
|
813
|
+
|
|
814
|
+
const mutation = activeOwnerMutationStatus()
|
|
815
|
+
const deadOwnerSocket = ownerClient === ownerMutationClient && ownerClientPid === ownerMutationPid ? ownerClient : undefined
|
|
816
|
+
const diagnostic = {
|
|
817
|
+
code: "OWNER_MUTATION_RECLAIMED",
|
|
818
|
+
event: "owner-mutation-reclaimed",
|
|
819
|
+
mutationId: ownerMutationId,
|
|
820
|
+
operation: mutation?.operation || "unnamed operation",
|
|
821
|
+
pid: ownerMutationPid,
|
|
822
|
+
reason: "owner mutation holder process is no longer alive"
|
|
823
|
+
}
|
|
824
|
+
|
|
825
|
+
process.stderr.write(`${JSON.stringify({at: new Date().toISOString(), ...diagnostic})}\n`)
|
|
826
|
+
broadcast(diagnostic)
|
|
827
|
+
clearOwnerMutationLease()
|
|
828
|
+
if (deadOwnerSocket) releaseClosedOwner(deadOwnerSocket)
|
|
829
|
+
return true
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
/**
|
|
833
|
+
* Checks whether a process still exists without signaling it.
|
|
834
|
+
* @param {number} pid - Process id.
|
|
835
|
+
* @returns {boolean} Whether the process exists.
|
|
836
|
+
*/
|
|
837
|
+
function isProcessAlive(pid) {
|
|
838
|
+
try {
|
|
839
|
+
process.kill(pid, 0)
|
|
840
|
+
return true
|
|
841
|
+
} catch (error) {
|
|
842
|
+
return Boolean(error && typeof error === "object" && "code" in error && error.code === "EPERM")
|
|
843
|
+
}
|
|
844
|
+
}
|
|
845
|
+
|
|
775
846
|
/** @returns {void} */
|
|
776
847
|
function clearOwnerMutationLease() {
|
|
777
848
|
if (ownerMutationLeaseTimer) clearTimeout(ownerMutationLeaseTimer)
|
package/src/release-group.js
CHANGED
|
@@ -23,7 +23,7 @@ const DRAIN_SAMPLE_WINDOW_MS = 20000
|
|
|
23
23
|
* @typedef {{cpuDeltaMs: number | undefined, reason: string, sampleWindowMs: number, status: import("./drain-policy.js").DrainProgress}} ReleaseDrainStatus
|
|
24
24
|
* @typedef {{http: number, websocket: number}} ReleaseConnections
|
|
25
25
|
* @typedef {{activatedAt: string | undefined, connectionCount: number, connections: ReleaseConnections, drain: ReleaseDrainStatus | undefined, drainStartedAt: string | undefined, ports: Record<string, number>, processes: import("./managed-process.js").ManagedProcessStatus[], releaseId: string, releasePath: string, retirementError: string | undefined, revision: string, state: ReleaseState, stoppedAt: string | undefined}} ReleaseStatus
|
|
26
|
-
* @typedef {{count?: number, guardianKey?: string, index?: number, instanceId?: string, shouldRestart?: () => boolean}} BuildProcessOptions
|
|
26
|
+
* @typedef {{count?: number, guardianKey?: string, index?: number, instanceId?: string, shouldRestart?: () => boolean, shouldStop?: () => boolean}} BuildProcessOptions
|
|
27
27
|
* @typedef {{deadlineMs: number, dependencyWaitMs: number, elapsedMs: number | undefined, failedProcessId: string, probes: number | undefined, stage: "dependency" | "health"}} StartupFailureDiagnostics
|
|
28
28
|
*/
|
|
29
29
|
|
|
@@ -507,6 +507,15 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
507
507
|
this.clearDrainLivenessSamples()
|
|
508
508
|
}
|
|
509
509
|
|
|
510
|
+
/** Stops only the proxied processes when a failed compensation cannot restore jobs. */
|
|
511
|
+
async stopProxyTraffic() {
|
|
512
|
+
const proxiedIds = new Set(this.config.processes.filter((processConfig) => processConfig.policy === "proxied").map((processConfig) => processConfig.id))
|
|
513
|
+
|
|
514
|
+
await Promise.allSettled([...this.processes.entries()]
|
|
515
|
+
.filter(([id]) => proxiedIds.has(id.replace(/#\d+$/, "")))
|
|
516
|
+
.map(([, processInstance]) => processInstance.stop()))
|
|
517
|
+
}
|
|
518
|
+
|
|
510
519
|
/**
|
|
511
520
|
* Restarts only the exact processes reconstructed for a committed generation.
|
|
512
521
|
* The caller must prove the durable transition identity before using this path.
|
|
@@ -625,6 +634,10 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
625
634
|
...this.baseEnvironment(processConfig, {count, index}),
|
|
626
635
|
...renderedEnv
|
|
627
636
|
}
|
|
637
|
+
const dependencyRetired = () => processConfig.dependsOn.some((dependencyId) => {
|
|
638
|
+
const dependency = this.processes.get(dependencyId)
|
|
639
|
+
return dependency?.lifecycleRole === "retired" || dependency?.status().lifecycleRole === "retired"
|
|
640
|
+
})
|
|
628
641
|
|
|
629
642
|
return {
|
|
630
643
|
command: renderTemplate(processConfig.command, context),
|
|
@@ -637,7 +650,15 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
637
650
|
outputLines: processConfig.outputLines,
|
|
638
651
|
restart: processConfig.restart,
|
|
639
652
|
restartDelayMs: processConfig.restartDelayMs,
|
|
640
|
-
shouldRestart: options.shouldRestart || (() =>
|
|
653
|
+
shouldRestart: options.shouldRestart || (() => {
|
|
654
|
+
if (this.state !== "active" && this.state !== "starting") return false
|
|
655
|
+
|
|
656
|
+
// A companion whose same-generation service dependency has retired can never
|
|
657
|
+
// satisfy its startup gate again. Treat that dependency lifecycle as terminal
|
|
658
|
+
// for this process so an exit does not become an endless supervisor restart loop.
|
|
659
|
+
return !dependencyRetired()
|
|
660
|
+
}),
|
|
661
|
+
shouldStop: options.shouldStop || dependencyRetired,
|
|
641
662
|
stopSignal: processConfig.stopSignal,
|
|
642
663
|
stopTimeoutMs: processConfig.gracefulStopMs
|
|
643
664
|
}
|
|
@@ -894,7 +915,8 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
894
915
|
return
|
|
895
916
|
}
|
|
896
917
|
|
|
897
|
-
|
|
918
|
+
const retirementTimeoutMs = config.proxy.nonBlockingDrainTimeoutMs
|
|
919
|
+
await Promise.allSettled(connectionDependent.map((processInstance) => processInstance.stop({retirementTimeoutMs})))
|
|
898
920
|
const nonBlockingResults = await Promise.allSettled(nonBlockingStops.map(({promise}) => promise))
|
|
899
921
|
nonBlockingResults.forEach((result, index) => {
|
|
900
922
|
if (result.status === "fulfilled" && this.processes.get(nonBlockingStops[index].id)?.retirementTimedOut) {
|
|
@@ -907,7 +929,7 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
907
929
|
})
|
|
908
930
|
}
|
|
909
931
|
})
|
|
910
|
-
await Promise.allSettled(handoffServices.map((processInstance) => processInstance.stop()))
|
|
932
|
+
await Promise.allSettled(handoffServices.map((processInstance) => processInstance.stop({retirementTimeoutMs})))
|
|
911
933
|
|
|
912
934
|
// Last act of the retirement transition, before the release is recorded stopped: whatever
|
|
913
935
|
// is still alive here is the residue, and the bracket pair above can now say whether it is
|
|
@@ -942,7 +964,14 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
942
964
|
this.drainStartedAt = new Date().toISOString()
|
|
943
965
|
}
|
|
944
966
|
|
|
967
|
+
// Refresh the live process definitions with the current retirement policy,
|
|
968
|
+
// but retain this generation's exact config identity for owner recovery.
|
|
969
|
+
// Replacing the stored generation config here would make a still-running
|
|
970
|
+
// singleton from the previous release fail the guardian provenance fence.
|
|
971
|
+
const generationConfig = this.config
|
|
972
|
+
|
|
945
973
|
this.refreshProcessDefinitions(config)
|
|
974
|
+
this.config = generationConfig
|
|
946
975
|
const generationIds = new Set([...this.handoffServiceIds, ...this.nonBlockingDrainIds])
|
|
947
976
|
const results = await Promise.allSettled([...this.processes.entries()]
|
|
948
977
|
.filter(([id]) => generationIds.has(id))
|
|
@@ -1027,6 +1056,9 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
1027
1056
|
* @returns {{classification: ReleaseDrainStatus, currentSample: DrainLivenessSample}} The observation the decision acts on, and what the pair says.
|
|
1028
1057
|
*/
|
|
1029
1058
|
observeRetirement({nowMs, previousSample}) {
|
|
1059
|
+
const drainBudgetMs = this.config.proxy.drainTimeoutMs
|
|
1060
|
+
const drainElapsedMs = this.drainStartedAt === undefined ? 0 : nowMs - Date.parse(this.drainStartedAt)
|
|
1061
|
+
|
|
1030
1062
|
if (previousSample === undefined && this.connectionCount > 0) {
|
|
1031
1063
|
const currentSample = this.drainNewestSample ?? {atMs: nowMs, processes: []}
|
|
1032
1064
|
|
|
@@ -1034,6 +1066,8 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
1034
1066
|
classification: classifyDrainProgress({
|
|
1035
1067
|
connectionCount: this.connectionCount,
|
|
1036
1068
|
currentSample,
|
|
1069
|
+
drainBudgetMs,
|
|
1070
|
+
drainElapsedMs,
|
|
1037
1071
|
minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
|
|
1038
1072
|
previousSample: undefined
|
|
1039
1073
|
}),
|
|
@@ -1056,6 +1090,8 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
1056
1090
|
classification: classifyDrainProgress({
|
|
1057
1091
|
connectionCount: this.connectionCount,
|
|
1058
1092
|
currentSample,
|
|
1093
|
+
drainBudgetMs,
|
|
1094
|
+
drainElapsedMs,
|
|
1059
1095
|
minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
|
|
1060
1096
|
previousSample: priorSample
|
|
1061
1097
|
}),
|
package/tensorbuzz.yml
CHANGED
|
@@ -18,4 +18,6 @@ paths:
|
|
|
18
18
|
before_script:
|
|
19
19
|
- npm install
|
|
20
20
|
script:
|
|
21
|
-
|
|
21
|
+
# Rollbridge's Unix control/guardian sockets must stay below the platform path limit;
|
|
22
|
+
# the TensorBuzz runner's workspace TMPDIR is deliberately long.
|
|
23
|
+
- TMPDIR=/tmp npm run all-checks
|
|
@@ -35,6 +35,8 @@ function sample(options = {}) {
|
|
|
35
35
|
* @param {object} [options] - Classification inputs.
|
|
36
36
|
* @param {number} [options.connectionCount] - Live connections owned by the generation.
|
|
37
37
|
* @param {import("../src/drain-policy.js").DrainLivenessSample} [options.current] - Newest observation.
|
|
38
|
+
* @param {number} [options.drainBudgetMs] - Configured drain budget.
|
|
39
|
+
* @param {number} [options.drainElapsedMs] - Time since the drain started.
|
|
38
40
|
* @param {import("../src/drain-policy.js").DrainLivenessSample | undefined} [options.previous] - Older observation.
|
|
39
41
|
* @returns {import("../src/drain-policy.js").DrainClassification} Classification.
|
|
40
42
|
*/
|
|
@@ -42,6 +44,8 @@ function classify(options = {}) {
|
|
|
42
44
|
return classifyDrainProgress({
|
|
43
45
|
connectionCount: options.connectionCount ?? 0,
|
|
44
46
|
currentSample: options.current ?? sample(),
|
|
47
|
+
drainBudgetMs: options.drainBudgetMs ?? 0,
|
|
48
|
+
drainElapsedMs: options.drainElapsedMs ?? 0,
|
|
45
49
|
minSampleWindowMs: WINDOW_MS,
|
|
46
50
|
previousSample: options.previous
|
|
47
51
|
})
|
|
@@ -71,6 +75,30 @@ test("a generation that still owns connections is draining, never wedged", () =>
|
|
|
71
75
|
expect(result.reason).toBe("3 live connections still owned by the generation")
|
|
72
76
|
})
|
|
73
77
|
|
|
78
|
+
test("a live connection past the configured drain budget is reported past the deadline, not progressing", () => {
|
|
79
|
+
const result = classify({connectionCount: 1, current: sample({liveChildCount: 0}), drainBudgetMs: 60000, drainElapsedMs: 61000})
|
|
80
|
+
|
|
81
|
+
expect(result.status).toBe("timedOut")
|
|
82
|
+
expect(result.reason).toBe("drain budget 60000ms exceeded by 1000ms with 1 live connection still owned by the generation")
|
|
83
|
+
})
|
|
84
|
+
|
|
85
|
+
test("a live connection inside the configured drain budget is still a legitimate drain", () => {
|
|
86
|
+
const result = classify({connectionCount: 1, current: sample({liveChildCount: 0}), drainBudgetMs: 60000, drainElapsedMs: 30000})
|
|
87
|
+
|
|
88
|
+
expect(result.status).toBe("progressing")
|
|
89
|
+
expect(result.reason).toBe("1 live connection still owned by the generation")
|
|
90
|
+
})
|
|
91
|
+
|
|
92
|
+
test("the drain deadline names the deadline and never escalates", () => {
|
|
93
|
+
const currentSample = sample({liveChildCount: 0})
|
|
94
|
+
const classification = classify({connectionCount: 1, current: currentSample, drainBudgetMs: 60000, drainElapsedMs: 61000})
|
|
95
|
+
const decision = decideWedgedEscalation({classification, currentSample, minSampleWindowMs: WINDOW_MS})
|
|
96
|
+
|
|
97
|
+
expect(decision.decision).toBe("report")
|
|
98
|
+
expect(decision.reason).toBe("no escalation: drain is timedOut (drain budget 60000ms exceeded by 1000ms with 1 live connection still owned by the generation)")
|
|
99
|
+
expect(decision.targets).toEqual([])
|
|
100
|
+
})
|
|
101
|
+
|
|
74
102
|
test("a process still inside its retirement budget is draining", () => {
|
|
75
103
|
const result = classify({current: sample({liveChildCount: 0, retirementTimedOut: false})})
|
|
76
104
|
|
|
@@ -7,19 +7,39 @@ import os from "node:os"
|
|
|
7
7
|
import path from "node:path"
|
|
8
8
|
import {describe, expect, test} from "@velocious/testing"
|
|
9
9
|
import {fileURLToPath} from "node:url"
|
|
10
|
-
import
|
|
10
|
+
import {createRequire} from "node:module"
|
|
11
|
+
import GuardianClient, {usableSocketPath} from "../src/guardian-client.js"
|
|
11
12
|
import {waitForProcessExit} from "./support/process.js"
|
|
12
13
|
|
|
13
14
|
describe("guardian-client", () => {
|
|
14
15
|
|
|
16
|
+
const commonJsRequire = createRequire(import.meta.url)
|
|
17
|
+
const packageVersion = /** @type {{version: string}} */ (commonJsRequire("../package.json")).version
|
|
15
18
|
const legacyGuardianPath = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures", "pre-split3-process-guardian.js")
|
|
16
19
|
const recoveryOwnerPath = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures", "guardian-recovery-owner.js")
|
|
17
20
|
|
|
21
|
+
test("guardian maps an overlong configured socket path to a stable short path", async () => {
|
|
22
|
+
const root = await fs.mkdtemp(path.join(os.tmpdir(), `rollbridge-${"x".repeat(120)}-`))
|
|
23
|
+
const configuredPath = path.join(root, "guardian.sock")
|
|
24
|
+
const client = new GuardianClient({socketPath: configuredPath, token: "long-path-capability"})
|
|
25
|
+
|
|
26
|
+
try {
|
|
27
|
+
expect(Buffer.byteLength(configuredPath)).toBeGreaterThan(107)
|
|
28
|
+
expect(Buffer.byteLength(client.socketPath)).toBeLessThanOrEqual(107)
|
|
29
|
+
expect(client.socketPath).not.toBe(configuredPath)
|
|
30
|
+
await client.launch()
|
|
31
|
+
expect(await client.capabilities()).toEqual({daemonRecovery: 1, generationReactivation: 1, guardianVersion: packageVersion})
|
|
32
|
+
} finally {
|
|
33
|
+
await cleanupGuardian({client, root})
|
|
34
|
+
}
|
|
35
|
+
})
|
|
36
|
+
|
|
18
37
|
test("guardian bootstrap capability is absent from process argv", async () => {
|
|
19
38
|
const fixture = await createGuardian()
|
|
39
|
+
const packageVersion = JSON.parse(await fs.readFile(path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "package.json"), "utf8")).version
|
|
20
40
|
|
|
21
41
|
try {
|
|
22
|
-
expect(await fixture.client.capabilities()).toEqual({daemonRecovery: 1, generationReactivation: 1})
|
|
42
|
+
expect(await fixture.client.capabilities()).toEqual({daemonRecovery: 1, generationReactivation: 1, guardianVersion: packageVersion})
|
|
23
43
|
const commandLine = await fs.readFile(`/proc/${fixture.client.pid}/cmdline`, "utf8")
|
|
24
44
|
const environment = await fs.readFile(`/proc/${fixture.client.pid}/environ`, "utf8")
|
|
25
45
|
const status = await fs.readFile(`/proc/${fixture.client.pid}/status`, "utf8")
|
|
@@ -90,6 +110,30 @@ test("an owner mutation that outruns its lease is released and the holder report
|
|
|
90
110
|
}
|
|
91
111
|
})
|
|
92
112
|
|
|
113
|
+
test("a dead owner-mutation holder is reclaimed and the next mutation is admitted", async () => {
|
|
114
|
+
const fixture = await createGuardian()
|
|
115
|
+
const holder = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"])
|
|
116
|
+
|
|
117
|
+
try {
|
|
118
|
+
const reclaimed = fixture.client.waitForEvent("owner-mutation-reclaimed")
|
|
119
|
+
await fixture.client.beginOwnerMutation("restart", {ownerPid: holder.pid, waitingFor: "a process that will die"})
|
|
120
|
+
if (!holder.pid) throw new Error("Missing mutation holder PID")
|
|
121
|
+
holder.kill("SIGKILL")
|
|
122
|
+
await once(holder, "exit")
|
|
123
|
+
|
|
124
|
+
const successor = await fixture.client.beginOwnerMutation("deploy", {ownerPid: process.pid, waitingFor: "candidate activation"})
|
|
125
|
+
const event = await reclaimed
|
|
126
|
+
|
|
127
|
+
expect(typeof successor).toBe("string")
|
|
128
|
+
expect(event).toEqual(expect.objectContaining({code: "OWNER_MUTATION_RECLAIMED", event: "owner-mutation-reclaimed", operation: "restart", pid: holder.pid, reason: "owner mutation holder process is no longer alive"}))
|
|
129
|
+
expect((await fixture.client.guardianStatus()).version).toBe(packageVersion)
|
|
130
|
+
await fixture.client.endOwnerMutation(successor)
|
|
131
|
+
} finally {
|
|
132
|
+
if (holder.exitCode === null && holder.signalCode === null) holder.kill("SIGKILL")
|
|
133
|
+
await cleanupGuardian(fixture)
|
|
134
|
+
}
|
|
135
|
+
})
|
|
136
|
+
|
|
93
137
|
test("guardian lets the owner re-register a key whose registered child is gone, and still fences a live one", async () => {
|
|
94
138
|
const fixture = await createGuardian()
|
|
95
139
|
const liveInstance = fixture.client.process("live-generation", definition("live-generation"))
|
|
@@ -912,7 +956,7 @@ test("replacement staging rejects owner state published after prepare", async ()
|
|
|
912
956
|
const prepared = await candidate.prepareOwnerReplacement(authority, nextAuthority)
|
|
913
957
|
|
|
914
958
|
await fixture.client.publishOwnerState({authority, snapshot: {activeReleaseId: "v2"}})
|
|
915
|
-
await expect(candidate.stageOwnerReplacement(prepared.replacementId, {authority: nextAuthority, snapshot: {activeReleaseId: "v1"}})).rejects.toThrow(/owner state changed after prepare/i)
|
|
959
|
+
await expect(candidate.stageOwnerReplacement(prepared.replacementId, {authority: nextAuthority, snapshot: {activeReleaseId: "v1"}})).rejects.toThrow(/owner state changed after prepare \(prepared revision 1, current revision 2\)/i)
|
|
916
960
|
await candidate.abortOwnerReplacement(prepared.replacementId)
|
|
917
961
|
const fresh = await candidate.prepareOwnerReplacement(authority, nextAuthority)
|
|
918
962
|
|
|
@@ -1748,7 +1792,7 @@ test("a legacy bridge remains discoverable when its candidate disconnects after
|
|
|
1748
1792
|
let replacement
|
|
1749
1793
|
|
|
1750
1794
|
try {
|
|
1751
|
-
const identity = {socketPath: path.join(fixture.root, "guardian-v2.sock"), token: "candidate-guardian-capability"}
|
|
1795
|
+
const identity = {socketPath: usableSocketPath(path.join(fixture.root, "guardian-v2.sock")), token: "candidate-guardian-capability"}
|
|
1752
1796
|
|
|
1753
1797
|
upgraded = await fixture.client.upgradeLegacyGuardian({ownerState, ...identity})
|
|
1754
1798
|
const prepared = await upgraded.prepareOwnerReplacement(authority, nextAuthority)
|
|
@@ -1795,7 +1839,7 @@ async function createGuardian() {
|
|
|
1795
1839
|
async function createLegacyGuardian() {
|
|
1796
1840
|
const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-legacy-guardian-"))
|
|
1797
1841
|
const token = "legacy-guardian-capability"
|
|
1798
|
-
const socketPath = path.join(root, "guardian.sock")
|
|
1842
|
+
const socketPath = usableSocketPath(path.join(root, "guardian.sock"))
|
|
1799
1843
|
const child = spawn(process.execPath, [legacyGuardianPath, socketPath], {detached: true, stdio: ["ignore", "ignore", "ignore", "ipc"]})
|
|
1800
1844
|
|
|
1801
1845
|
await new Promise((resolve, reject) => {
|
|
@@ -541,7 +541,9 @@ test("a retirement budget caps a finite graceful window instead of being outvote
|
|
|
541
541
|
// 4,5 h and killed the in-flight builds the consumer said not to interrupt.
|
|
542
542
|
expect(deadlines.map((entry) => entry.phase)).toEqual(["signal", "wait"])
|
|
543
543
|
for (const entry of deadlines) {
|
|
544
|
-
|
|
544
|
+
// The deadline starts at stop entry; a millisecond clock tick between this test's marker
|
|
545
|
+
// and the async stop call may shave 1 ms from the observed offset.
|
|
546
|
+
expect(entry.msFromStopStart).toBeGreaterThanOrEqual(0)
|
|
545
547
|
expect(entry.msFromStopStart).toBeLessThan(1000)
|
|
546
548
|
}
|
|
547
549
|
expect(managed.retirementTimedOut).toBe(true)
|