rollbridge 0.1.68 → 0.1.69
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -0
- package/changelog.d/20261116-guardian-version-handoff.md +37 -0
- package/changelog.d/20261116-reclaim-dead-owner-mutation.md +7 -0
- package/changelog.d/20261117-activation-path-drain-deadline.md +35 -0
- package/changelog.d/20261117-stop-companion-after-retired-dependency.md +7 -0
- package/docs/cli.md +14 -7
- package/docs/config.md +8 -4
- package/docs/troubleshooting.md +27 -0
- package/docs/workers.md +1 -0
- package/examples/tensorbuzz.com.js +2 -1
- package/package.json +1 -1
- package/src/daemon.js +144 -35
- package/src/doctor.js +7 -0
- package/src/drain-policy.js +18 -4
- package/src/guardian-client.js +54 -6
- package/src/managed-process.js +14 -5
- package/src/process-guardian.js +75 -4
- package/src/release-group.js +30 -2
- package/tensorbuzz.yml +3 -1
- package/test/drain-policy.test.js +28 -0
- package/test/guardian-client.test.js +48 -4
- package/test/managed-process.test.js +3 -1
- package/test/owner-recovery.test.js +43 -4
- package/test/owner-replacement.test.js +94 -9
- package/test/release-group.test.js +57 -0
- package/test/rollbridge.test.js +84 -9
- package/test/shutdown-completion.test.js +52 -0
package/src/daemon.js
CHANGED
|
@@ -11,7 +11,7 @@ import {loadConfig} from "./config.js"
|
|
|
11
11
|
import {openControlSession} from "./control-client.js"
|
|
12
12
|
import {admitGenerationRelease} from "./drain-policy.js"
|
|
13
13
|
import EventLog from "./event-log.js"
|
|
14
|
-
import GuardianClient from "./guardian-client.js"
|
|
14
|
+
import GuardianClient, {guardianVersionFromProcess, usableSocketPath} from "./guardian-client.js"
|
|
15
15
|
import {childRecordIsGone} from "./managed-process.js"
|
|
16
16
|
import ReleaseGroup from "./release-group.js"
|
|
17
17
|
import {clearState, isProcessAlive, liveProcesses, readState, writeState} from "./state-store.js"
|
|
@@ -25,10 +25,11 @@ const STATE_PERSIST_INTERVAL_MS = 5000
|
|
|
25
25
|
* @typedef {{releaseId?: string, releasePath: string, revision?: string}} DeployArgs
|
|
26
26
|
* @typedef {{attestation?: string, releaseId: string, releasePath: string, revision: string}} BootstrapIdentity
|
|
27
27
|
* @typedef {{id: string, process: import("./managed-process.js").ManagedProcessStatus}} ProcessStatus
|
|
28
|
-
* @typedef {{disruptive: true, mode: "legacy-first-upgrade", reason: string}} OwnerTransition
|
|
28
|
+
* @typedef {{disruptive: true, mode: "guardian-version-handoff" | "legacy-first-upgrade", reason: string}} OwnerTransition
|
|
29
|
+
* @typedef {"guardian-version-drift" | "legacy-protocol"} GuardianHandoffProtocol
|
|
29
30
|
* @typedef {"candidate_ready" | "retiring_previous" | "previous_retired" | "activating_candidate" | "restoring_previous" | "retiring_failed_candidate" | "degraded_active" | "committed_pending" | "committed" | "restoring_committed"} GenerationTransitionPhase
|
|
30
31
|
* @typedef {{activationError?: string, activationLifecycle?: boolean, candidateReleaseId: string, candidateReleasePath: string, candidateRevision: string, compensationError?: string, configDigest: string, degradedIncumbent?: boolean, error?: string, journalRevision?: number, phase: GenerationTransitionPhase, previousReleaseId: string | null, startedAt: string, updatedAt: string}} GenerationTransition
|
|
31
|
-
* @typedef {{activeReleaseId: string | null, application: string, bootstrap: BootstrapIdentity | undefined, control: import("./config.js").ControlConfig, daemonPid: number, daemonRuntime: import("./daemon-runtime.js").DaemonRuntimeIdentity | undefined, generationTransition?: GenerationTransition, ownerRecovery: {configDigest: string, ready: boolean} | undefined, ownerTransition?: OwnerTransition, orphans: {id: string, pid: number, releaseId: string | null}[], proxy: {host: string, port: number | undefined, upstreamHost: string}, releaseReferences: {releaseId: string, releasePath: string}[], releases: import("./release-group.js").ReleaseStatus[], services: ProcessStatus[], singletons: ProcessStatus[]}} DaemonStatus
|
|
32
|
+
* @typedef {{activeReleaseId: string | null, application: string, bootstrap: BootstrapIdentity | undefined, control: import("./config.js").ControlConfig, daemonPid: number, daemonRuntime: import("./daemon-runtime.js").DaemonRuntimeIdentity | undefined, generationTransition?: GenerationTransition, guardian?: {guardianPid: number | null, ownerMutation: Record<string, JsonValue> | null, version: string | null}, ownerRecovery: {configDigest: string, ready: boolean} | undefined, ownerTransition?: OwnerTransition, orphans: {id: string, pid: number, releaseId: string | null}[], proxy: {host: string, port: number | undefined, upstreamHost: string}, releaseReferences: {releaseId: string, releasePath: string}[], releases: import("./release-group.js").ReleaseStatus[], services: ProcessStatus[], singletons: ProcessStatus[]}} DaemonStatus
|
|
32
33
|
* @typedef {Omit<import("./managed-process.js").ManagedProcessStatus, "logs">} ManagedProcessStatusWithoutLogs
|
|
33
34
|
* @typedef {Omit<import("./release-group.js").ReleaseStatus, "processes"> & {processes: ManagedProcessStatusWithoutLogs[]}} ReleaseStatusWithoutLogs
|
|
34
35
|
* @typedef {Omit<ProcessStatus, "process"> & {process: ManagedProcessStatusWithoutLogs}} ProcessStatusWithoutLogs
|
|
@@ -37,7 +38,7 @@ const STATE_PERSIST_INTERVAL_MS = 5000
|
|
|
37
38
|
* @typedef {{configDigest: string, format: number, guardian: {pid?: number, socketPath: string, token: string}, reconnectGraceMs: number}} OwnerRecoveryMetadata
|
|
38
39
|
* @typedef {DaemonStatus & {recovery: OwnerRecoveryMetadata, serviceReleaseIds?: Record<string, string>, singletonReleaseIds?: Record<string, string>}} OwnerRecoverySnapshot
|
|
39
40
|
* @typedef {{authority: JsonValue, config: import("./config.js").RollbridgeConfig, listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>, listenerSourceId?: string, recovery?: {command: JsonValue, reconnectGraceMs: number, startupTimeoutMs: number}, releaseConfigs?: Record<string, import("./config.js").RollbridgeConfig>, retiredListenerHandoff?: number, serviceReleaseIds?: Record<string, string>, singletonReleaseIds?: Record<string, string>, snapshot: OwnerRecoverySnapshot}} PrivateOwnerState
|
|
40
|
-
* @typedef {{boundaryCrossed: boolean, incumbentControl?: Awaited<ReturnType<typeof openControlSession>>, incumbentStartTime: string, legacyGuardian: GuardianClient, legacyInventory: {key: string, provenance: string}[], legacyPrepared?: {ownerState: JsonValue, replacementId: string}, legacySnapshot: OwnerRecoverySnapshot, prepared: {ownerState: JsonValue, replacementId: string}, recoverySnapshot: OwnerRecoverySnapshot}} LegacyOwnerBridge
|
|
41
|
+
* @typedef {{boundaryCrossed: boolean, guardianVersion?: string, incumbentControl?: Awaited<ReturnType<typeof openControlSession>>, incumbentStartTime: string, legacyGuardian: GuardianClient, legacyInventory: {key: string, provenance: string}[], legacyPrepared?: {ownerState: JsonValue, replacementId: string}, legacySnapshot: OwnerRecoverySnapshot, prepared: {ownerState: JsonValue, replacementId: string}, protocol: GuardianHandoffProtocol, recoverySnapshot: OwnerRecoverySnapshot}} LegacyOwnerBridge
|
|
41
42
|
*/
|
|
42
43
|
|
|
43
44
|
export default class RollbridgeDaemon {
|
|
@@ -206,7 +207,7 @@ export default class RollbridgeDaemon {
|
|
|
206
207
|
if (!snapshot || snapshot.recovery?.guardian || !exactBootstrap) throw new Error("Retired owner reset requires an exact committed bootstrap with missing guardian identity")
|
|
207
208
|
const control = await inspectControlSocket(this.config.control.path)
|
|
208
209
|
if (control.alive) throw new Error(`Retired owner reset refuses responsive control owner at ${this.config.control.path}`)
|
|
209
|
-
this.guardianIdentity = {socketPath: `${this.statePath}.guardian.sock
|
|
210
|
+
this.guardianIdentity = {socketPath: usableSocketPath(`${this.statePath}.guardian.sock`), token: crypto.randomBytes(32).toString("hex")}
|
|
210
211
|
this.guardian = new GuardianClient(this.guardianIdentity)
|
|
211
212
|
await this.guardian.launch()
|
|
212
213
|
this.guardianIdentity.pid = this.guardian.pid
|
|
@@ -224,10 +225,12 @@ export default class RollbridgeDaemon {
|
|
|
224
225
|
|
|
225
226
|
if (snapshot && !recovery) throw new Error(`Owner recovery state ${this.statePath} is missing durable guardian identity; refusing to overwrite it.`)
|
|
226
227
|
|
|
227
|
-
const guardianIdentity = recovery?.guardian
|
|
228
|
-
socketPath:
|
|
229
|
-
|
|
230
|
-
|
|
228
|
+
const guardianIdentity = recovery?.guardian
|
|
229
|
+
? {...recovery.guardian, socketPath: usableSocketPath(recovery.guardian.socketPath)}
|
|
230
|
+
: {
|
|
231
|
+
socketPath: usableSocketPath(`${this.statePath}.guardian.sock`),
|
|
232
|
+
token: crypto.randomBytes(32).toString("hex")
|
|
233
|
+
}
|
|
231
234
|
this.guardianIdentity = guardianIdentity
|
|
232
235
|
this.guardian = new GuardianClient(guardianIdentity)
|
|
233
236
|
if (recovery) {
|
|
@@ -511,7 +514,7 @@ export default class RollbridgeDaemon {
|
|
|
511
514
|
const persisted = state && typeof state === "object" && !Array.isArray(state) ? /** @type {OwnerRecoverySnapshot} */ (state) : undefined
|
|
512
515
|
|
|
513
516
|
if (!persisted?.recovery?.guardian) throw new Error(`No committed durable owner transaction was found at ${this.statePath}`)
|
|
514
|
-
this.guardianIdentity = persisted.recovery.guardian
|
|
517
|
+
this.guardianIdentity = {...persisted.recovery.guardian, socketPath: usableSocketPath(persisted.recovery.guardian.socketPath)}
|
|
515
518
|
this.guardian = new GuardianClient(this.guardianIdentity)
|
|
516
519
|
await this.guardian.connect()
|
|
517
520
|
this.watchOwnerReplacementEvents()
|
|
@@ -519,9 +522,16 @@ export default class RollbridgeDaemon {
|
|
|
519
522
|
configDigest: persisted.recovery.configDigest,
|
|
520
523
|
runtime: persisted.daemonRuntime ? {...persisted.daemonRuntime} : null
|
|
521
524
|
}
|
|
522
|
-
const
|
|
525
|
+
const guardianVersion = await this.retainedGuardianVersion()
|
|
526
|
+
const versionDrift = Boolean(this.runtime?.version && guardianVersion && guardianVersion !== this.runtime.version)
|
|
527
|
+
const replacementProtocol = versionDrift ? "legacy" : await this.guardian.ownerReplacementProtocol()
|
|
523
528
|
const legacyBridge = replacementProtocol === "legacy"
|
|
524
|
-
? await this.prepareLegacyOwnerReplacement({
|
|
529
|
+
? await this.prepareLegacyOwnerReplacement({
|
|
530
|
+
guardianVersion,
|
|
531
|
+
persisted,
|
|
532
|
+
persistedAuthority,
|
|
533
|
+
protocol: versionDrift ? "guardian-version-drift" : "legacy-protocol"
|
|
534
|
+
})
|
|
525
535
|
: undefined
|
|
526
536
|
const prepared = legacyBridge?.prepared ?? await this.guardian.prepareOwnerReplacement(persistedAuthority, this.ownerAuthority())
|
|
527
537
|
let preparedStatus
|
|
@@ -767,6 +777,33 @@ export default class RollbridgeDaemon {
|
|
|
767
777
|
}
|
|
768
778
|
}
|
|
769
779
|
|
|
780
|
+
/**
|
|
781
|
+
* Reads the retained guardian's package version without deciding the replacement protocol.
|
|
782
|
+
*
|
|
783
|
+
* The version is read from the guardian's own executable rather than its capabilities
|
|
784
|
+
* command: a pre-replacement guardian only answers that command once a replacement
|
|
785
|
+
* transaction is prepared, so asking here would move the probe ahead of the transaction
|
|
786
|
+
* every capability-gated guardian depends on. A guardian whose version cannot be read
|
|
787
|
+
* stays on the protocol classification path.
|
|
788
|
+
* @returns {Promise<string | undefined>} Retained guardian package version.
|
|
789
|
+
*/
|
|
790
|
+
async retainedGuardianVersion() {
|
|
791
|
+
if (!this.guardian) throw new Error("Guardian version reconciliation requires the durable process guardian")
|
|
792
|
+
return await guardianVersionFromProcess(this.guardian.pid)
|
|
793
|
+
}
|
|
794
|
+
|
|
795
|
+
/**
|
|
796
|
+
* Names the last-resort refusal for a guardian this daemon cannot hand off automatically.
|
|
797
|
+
* @param {string | undefined} guardianVersion - Retained guardian package version.
|
|
798
|
+
* @param {string} cause - Measured reason the automatic handoff cannot run.
|
|
799
|
+
* @returns {Error} Refusal naming both versions and the exact repair command.
|
|
800
|
+
*/
|
|
801
|
+
guardianVersionHandoffRefusal(guardianVersion, cause) {
|
|
802
|
+
const configArgument = this.configPath ? ` --config ${this.configPath}` : ""
|
|
803
|
+
|
|
804
|
+
return new Error(`The persistent Rollbridge guardian is version ${guardianVersion ?? "unknown"} while this daemon runs rollbridge ${this.runtime?.version ?? "unknown"}; ${cause}. Run "rollbridge shutdown${configArgument}" and then "rollbridge ensure-daemon${configArgument}" to replace the supervisor with this Rollbridge installation.`)
|
|
805
|
+
}
|
|
806
|
+
|
|
770
807
|
/** Requires the persistent guardian protocol which can restart an accepted daemon. */
|
|
771
808
|
async assertGuardianDaemonRecoveryCapability() {
|
|
772
809
|
if (!this.guardian) throw new Error("Owner recovery capability check requires the durable process guardian")
|
|
@@ -775,10 +812,18 @@ export default class RollbridgeDaemon {
|
|
|
775
812
|
try {
|
|
776
813
|
capabilities = await this.guardian.capabilities()
|
|
777
814
|
} catch (error) {
|
|
778
|
-
if (!(error instanceof Error) || !
|
|
815
|
+
if (!(error instanceof Error) || !isGuardianDaemonRecoveryDiagnostic(error.message)) throw error
|
|
779
816
|
throw new Error("The persistent Rollbridge guardian predates daemon recovery. Perform one explicit clean Rollbridge shutdown and restart before retrying this package upgrade.", {cause: error})
|
|
780
817
|
}
|
|
781
818
|
if (capabilities.daemonRecovery !== 1) throw new Error("The persistent Rollbridge guardian does not support the required daemon recovery protocol")
|
|
819
|
+
const clientVersion = this.runtime?.version
|
|
820
|
+
const guardianVersion = capabilities.guardianVersion
|
|
821
|
+
|
|
822
|
+
if (!clientVersion) return
|
|
823
|
+
if (!guardianVersion) throw this.guardianVersionHandoffRefusal(guardianVersion, "its version cannot be attested, so no handoff can be authorised")
|
|
824
|
+
if (guardianVersion !== clientVersion) {
|
|
825
|
+
throw this.guardianVersionHandoffRefusal(guardianVersion, "this startup path has no incumbent daemon to hand guardian ownership from")
|
|
826
|
+
}
|
|
782
827
|
}
|
|
783
828
|
|
|
784
829
|
/**
|
|
@@ -899,13 +944,15 @@ export default class RollbridgeDaemon {
|
|
|
899
944
|
}
|
|
900
945
|
|
|
901
946
|
/**
|
|
902
|
-
* Authenticates and prepares the
|
|
903
|
-
*
|
|
947
|
+
* Authenticates and prepares the disruptive guardian-upgrade bridge, for a guardian that predates
|
|
948
|
+
* the replacement protocol or whose package version drifted from this daemon's runtime.
|
|
949
|
+
* @param {{guardianVersion?: string, persisted: OwnerRecoverySnapshot, persistedAuthority: {configDigest: string, runtime: import("./daemon-runtime.js").DaemonRuntimeIdentity | null}, protocol: GuardianHandoffProtocol}} options - Bridge evidence and why the bridge is required.
|
|
904
950
|
* @returns {Promise<LegacyOwnerBridge>} Prepared bridge.
|
|
905
951
|
*/
|
|
906
|
-
async prepareLegacyOwnerReplacement({persisted, persistedAuthority}) {
|
|
952
|
+
async prepareLegacyOwnerReplacement({guardianVersion, persisted, persistedAuthority, protocol}) {
|
|
907
953
|
const legacyProcessKey = ownerSnapshotProcessKeys(persisted, persisted.singletonReleaseIds)[0]
|
|
908
954
|
const legacyGuardian = this.guardian
|
|
955
|
+
const versionHandoff = protocol === "guardian-version-drift"
|
|
909
956
|
let legacyPrepared
|
|
910
957
|
|
|
911
958
|
if (!legacyProcessKey) throw new Error("Legacy disruptive owner replacement requires an exact guardian-owned process registration in durable state")
|
|
@@ -938,12 +985,23 @@ export default class RollbridgeDaemon {
|
|
|
938
985
|
|
|
939
986
|
try {
|
|
940
987
|
if (persisted.recovery.configDigest !== this.ownerRecoveryConfigDigest()) {
|
|
988
|
+
if (versionHandoff) throw this.guardianVersionHandoffRefusal(guardianVersion, "the handoff keeps the incumbent config identity unchanged and this deploy changes config authority")
|
|
941
989
|
throw new Error("The one-time legacy guardian bridge requires the incumbent config identity unchanged; retry the config change after the bridge establishes split-3 authority")
|
|
942
990
|
}
|
|
943
|
-
if (!this.legacyIncumbentPid)
|
|
991
|
+
if (!this.legacyIncumbentPid) {
|
|
992
|
+
if (versionHandoff) throw this.guardianVersionHandoffRefusal(guardianVersion, "ensure-daemon reported no incumbent daemon to hand guardian ownership from")
|
|
993
|
+
throw new Error("The authenticated legacy guardian requires an exact incumbent PID from ensure-daemon for the disruptive bridge")
|
|
994
|
+
}
|
|
944
995
|
if (!this.guardianIdentity?.pid) throw new Error("The authenticated legacy guardian state is missing its exact guardian PID")
|
|
945
996
|
await verifyLegacyGuardianProcess(this.guardianIdentity.pid, this.guardianIdentity.socketPath)
|
|
946
|
-
|
|
997
|
+
let incumbentStartTime
|
|
998
|
+
|
|
999
|
+
try {
|
|
1000
|
+
incumbentStartTime = await verifyLegacyDaemonProcess(this.legacyIncumbentPid, this.configPath, persisted.control.path)
|
|
1001
|
+
} catch (error) {
|
|
1002
|
+
if (!versionHandoff) throw error
|
|
1003
|
+
throw this.guardianVersionHandoffRefusal(guardianVersion, `the incumbent daemon PID ${this.legacyIncumbentPid} is not the attested owner: ${error instanceof Error ? error.message : String(error)}`)
|
|
1004
|
+
}
|
|
947
1005
|
if (legacyPrepared) assertLegacyPrivateOwnerState(legacyPrepared.ownerState, persisted, persistedAuthority)
|
|
948
1006
|
try {
|
|
949
1007
|
incumbentControl = await openControlSession(persisted.control.path)
|
|
@@ -970,7 +1028,9 @@ export default class RollbridgeDaemon {
|
|
|
970
1028
|
|
|
971
1029
|
assertLegacyGuardianInventoryMembership(legacyInventory, /** @type {PrivateOwnerState} */ (ownerState))
|
|
972
1030
|
const upgradedIdentity = /** @type {{pid?: number, socketPath: string, token: string}} */ ({
|
|
973
|
-
socketPath:
|
|
1031
|
+
socketPath: usableSocketPath(versionHandoff
|
|
1032
|
+
? `${this.statePath}.guardian-${this.runtime?.version}.sock`
|
|
1033
|
+
: `${this.statePath}.split3-guardian.sock`),
|
|
974
1034
|
token: crypto.randomBytes(32).toString("hex")
|
|
975
1035
|
})
|
|
976
1036
|
|
|
@@ -985,12 +1045,15 @@ export default class RollbridgeDaemon {
|
|
|
985
1045
|
recovery: {...persisted.recovery, guardian: upgradedIdentity}
|
|
986
1046
|
})
|
|
987
1047
|
|
|
988
|
-
this.logger("
|
|
1048
|
+
this.logger("guardian upgrade bridge prepared", {
|
|
989
1049
|
guardianPid: upgradedIdentity.pid ?? null,
|
|
1050
|
+
guardianVersion: guardianVersion ?? "unknown",
|
|
990
1051
|
incumbentPid: this.legacyIncumbentPid,
|
|
991
|
-
|
|
1052
|
+
protocol,
|
|
1053
|
+
replacementId: prepared.replacementId,
|
|
1054
|
+
runtimeVersion: this.runtime?.version ?? "unknown"
|
|
992
1055
|
})
|
|
993
|
-
return {boundaryCrossed: false, incumbentControl, incumbentStartTime, legacyGuardian, legacyInventory, legacyPrepared, legacySnapshot: persisted, prepared, recoverySnapshot}
|
|
1056
|
+
return {boundaryCrossed: false, guardianVersion, incumbentControl, incumbentStartTime, legacyGuardian, legacyInventory, legacyPrepared, legacySnapshot: persisted, prepared, protocol, recoverySnapshot}
|
|
994
1057
|
} catch (upgradeError) {
|
|
995
1058
|
incumbentControl?.close()
|
|
996
1059
|
let cleanupError
|
|
@@ -1070,10 +1133,20 @@ export default class RollbridgeDaemon {
|
|
|
1070
1133
|
|
|
1071
1134
|
if (staged.committed) throw new Error("Legacy guardian committed replacement before the authenticated disruptive boundary")
|
|
1072
1135
|
}
|
|
1073
|
-
|
|
1136
|
+
const boundaryReason = bridge.protocol === "guardian-version-drift"
|
|
1137
|
+
? `retained guardian ${bridge.guardianVersion ?? "unknown"} predates this daemon runtime ${this.runtime?.version ?? "unknown"}`
|
|
1138
|
+
: "retained guardian and daemon lacked atomic replacement protocol"
|
|
1139
|
+
|
|
1140
|
+
this.logger("guardian upgrade bridge crossed", {
|
|
1074
1141
|
disruptive: true,
|
|
1142
|
+
guardianVersion: bridge.guardianVersion ?? "unknown",
|
|
1075
1143
|
incumbentPid: this.legacyIncumbentPid,
|
|
1076
|
-
|
|
1144
|
+
protocol: bridge.protocol,
|
|
1145
|
+
reason: boundaryReason,
|
|
1146
|
+
// The retained guardian keeps its own children; the boundary retires only the incumbent daemon.
|
|
1147
|
+
retainedGuardianPid: bridge.legacyGuardian.pid ?? null,
|
|
1148
|
+
retainedGuardianProcesses: bridge.legacyInventory.length,
|
|
1149
|
+
runtimeVersion: this.runtime?.version ?? "unknown"
|
|
1077
1150
|
})
|
|
1078
1151
|
await this.guardian.beginLegacyOwnerClaim(
|
|
1079
1152
|
bridge.prepared.replacementId,
|
|
@@ -1089,8 +1162,8 @@ export default class RollbridgeDaemon {
|
|
|
1089
1162
|
await this.guardian.completeLegacyOwnerClaim(bridge.prepared.replacementId)
|
|
1090
1163
|
this.ownerTransition = /** @type {OwnerTransition} */ ({
|
|
1091
1164
|
disruptive: true,
|
|
1092
|
-
mode: "legacy-first-upgrade",
|
|
1093
|
-
reason:
|
|
1165
|
+
mode: bridge.protocol === "guardian-version-drift" ? "guardian-version-handoff" : "legacy-first-upgrade",
|
|
1166
|
+
reason: boundaryReason
|
|
1094
1167
|
})
|
|
1095
1168
|
}
|
|
1096
1169
|
|
|
@@ -1406,7 +1479,18 @@ export default class RollbridgeDaemon {
|
|
|
1406
1479
|
}
|
|
1407
1480
|
|
|
1408
1481
|
if (commandName === "status") {
|
|
1409
|
-
|
|
1482
|
+
const status = this.status({includeLogs: statusIncludeLogs(data.includeLogs)})
|
|
1483
|
+
let guardian
|
|
1484
|
+
|
|
1485
|
+
try {
|
|
1486
|
+
const diagnostics = await this.guardian?.guardianStatus()
|
|
1487
|
+
|
|
1488
|
+
guardian = diagnostics ? {guardianPid: diagnostics.guardianPid, ownerMutation: diagnostics.ownerMutation, version: diagnostics.version} : {guardianPid: null, ownerMutation: null, version: null}
|
|
1489
|
+
} catch (error) {
|
|
1490
|
+
guardian = {error: error instanceof Error ? error.message : String(error), guardianPid: this.guardianIdentity?.pid ?? null, ownerMutation: null, version: null}
|
|
1491
|
+
}
|
|
1492
|
+
|
|
1493
|
+
return {...status, guardian}
|
|
1410
1494
|
}
|
|
1411
1495
|
|
|
1412
1496
|
if (commandName === "events") {
|
|
@@ -1956,6 +2040,8 @@ export default class RollbridgeDaemon {
|
|
|
1956
2040
|
if (!candidate) throw new Error(`Generation transition candidate ${transition.candidateReleaseId} is not retained`)
|
|
1957
2041
|
if (!previous) throw new Error(`Generation transition previous release ${transition.previousReleaseId} is not retained`)
|
|
1958
2042
|
if (this.activeRelease !== previous) throw new Error(`Previous release ${previous.releaseId} is not the authoritative proxy target`)
|
|
2043
|
+
/** @type {Promise<void> | undefined} */
|
|
2044
|
+
let candidateDrain
|
|
1959
2045
|
|
|
1960
2046
|
if (transition.phase === "activating_candidate") {
|
|
1961
2047
|
if (!transition.error && !transition.activationError) throw new Error("Candidate activation has no recorded failure to compensate")
|
|
@@ -1987,6 +2073,15 @@ export default class RollbridgeDaemon {
|
|
|
1987
2073
|
transition.compensationError = undefined
|
|
1988
2074
|
await this.updateGenerationTransition("restoring_previous")
|
|
1989
2075
|
this.logger("release generation compensation candidate retired", {releaseId: candidate.releaseId})
|
|
2076
|
+
|
|
2077
|
+
// The candidate's bounded drain starts here, before the restoration attempt. The only other
|
|
2078
|
+
// compensation drain runs after the success path, so when incumbent restoration throws the
|
|
2079
|
+
// candidate is left `draining` with a live process: measured 2026-11-17, candidate
|
|
2080
|
+
// `20261117063703` still held `web` pid 3201928 at 07:30Z after its compensation threw at
|
|
2081
|
+
// 06:42:31Z, and reached `release drained` only when a later compensation reached the
|
|
2082
|
+
// success path at 08:47:00Z. A retained generation holding live work is what refuses every
|
|
2083
|
+
// later deploy, and a failed candidate is never a restoration target.
|
|
2084
|
+
candidateDrain = this.drainAndPrune(candidate, candidate.config)
|
|
1990
2085
|
}
|
|
1991
2086
|
|
|
1992
2087
|
if (transition.phase === "restoring_previous") {
|
|
@@ -1998,10 +2093,12 @@ export default class RollbridgeDaemon {
|
|
|
1998
2093
|
|
|
1999
2094
|
transition.compensationError = failure
|
|
2000
2095
|
await this.failGenerationTransition(`Incumbent ${previous.releaseId} restoration failed: ${failure}`)
|
|
2096
|
+
await previous.stopProxyTraffic()
|
|
2001
2097
|
this.logger("release generation compensation restoration failed", {
|
|
2002
2098
|
activationError: transition.activationError,
|
|
2003
2099
|
error: failure,
|
|
2004
|
-
releaseId: previous.releaseId
|
|
2100
|
+
releaseId: previous.releaseId,
|
|
2101
|
+
traffic: "unavailable"
|
|
2005
2102
|
})
|
|
2006
2103
|
throw new Error(`incumbent ${previous.releaseId} restoration failed: ${failure}`, {cause: error})
|
|
2007
2104
|
}
|
|
@@ -2040,7 +2137,7 @@ export default class RollbridgeDaemon {
|
|
|
2040
2137
|
this.logger("release generation compensation checkpoint failed", {error: failure, releaseId: candidate.releaseId})
|
|
2041
2138
|
throw new Error(`compensation checkpoint clear failed: ${failure}`, {cause: error})
|
|
2042
2139
|
}
|
|
2043
|
-
void this.drainAndPrune(candidate, candidate.config)
|
|
2140
|
+
if (candidateDrain === undefined) void this.drainAndPrune(candidate, candidate.config)
|
|
2044
2141
|
this.logger("release generation compensation completed", result)
|
|
2045
2142
|
return result
|
|
2046
2143
|
}
|
|
@@ -3032,13 +3129,13 @@ export default class RollbridgeDaemon {
|
|
|
3032
3129
|
await captureShutdownError(cleanupErrors, "proxy close", async () => this.proxy.close())
|
|
3033
3130
|
const dependentStopResults = await Promise.allSettled([
|
|
3034
3131
|
...[...this.singletons.values()].map((processInstance) => processInstance.stop()),
|
|
3035
|
-
...[...this.startingReleases].map((release) => release.stop()),
|
|
3036
|
-
...[...this.releases.values()].map((release) => release.stop())
|
|
3132
|
+
...[...this.startingReleases].map((release) => release.stop({retirementTimeoutMs: this.config.proxy.nonBlockingDrainTimeoutMs})),
|
|
3133
|
+
...[...this.releases.values()].map((release) => release.stop({retirementTimeoutMs: this.config.proxy.nonBlockingDrainTimeoutMs}))
|
|
3037
3134
|
])
|
|
3038
3135
|
const serviceStopResults = await Promise.allSettled([...this.services.values()].map((processInstance) => processInstance.stop()))
|
|
3039
3136
|
const stopResults = [...dependentStopResults, ...serviceStopResults]
|
|
3040
3137
|
const guardian = this.guardian
|
|
3041
|
-
if (guardian) await captureShutdownError(cleanupErrors, "process guardian shutdown", () => guardian.shutdown())
|
|
3138
|
+
if (guardian) await captureShutdownError(cleanupErrors, "process guardian shutdown", () => guardian.shutdown({retirementTimeoutMs: this.config.proxy.nonBlockingDrainTimeoutMs}))
|
|
3042
3139
|
await captureShutdownError(cleanupErrors, "proxy server close", () => this.closeServer(this.proxyServer))
|
|
3043
3140
|
|
|
3044
3141
|
// Wait for any in-flight write first so it can't recreate or overwrite the final state (no
|
|
@@ -3315,6 +3412,15 @@ export function isLegacyGuardianPrepareDiagnostic(diagnostic) {
|
|
|
3315
3412
|
diagnostic === "Unknown guardian command: prepare-owner-replacement"
|
|
3316
3413
|
}
|
|
3317
3414
|
|
|
3415
|
+
/**
|
|
3416
|
+
* Reports the capability diagnostics that identify a guardian predating daemon recovery.
|
|
3417
|
+
* @param {string} diagnostic - Guardian error message.
|
|
3418
|
+
* @returns {boolean} Whether the guardian predates the daemon recovery capability.
|
|
3419
|
+
*/
|
|
3420
|
+
function isGuardianDaemonRecoveryDiagnostic(diagnostic) {
|
|
3421
|
+
return diagnostic === "Guardian capabilities requires a process key" || diagnostic === "Unknown guardian command: capabilities"
|
|
3422
|
+
}
|
|
3423
|
+
|
|
3318
3424
|
/**
|
|
3319
3425
|
* @param {DaemonStatus} snapshot - Serialized owner process snapshot.
|
|
3320
3426
|
* @param {Record<string, string>} [singletonReleaseIds] - Exact singleton owner releases.
|
|
@@ -3662,7 +3768,7 @@ export function releasesToPrune(releases, policy, now) {
|
|
|
3662
3768
|
}
|
|
3663
3769
|
|
|
3664
3770
|
/**
|
|
3665
|
-
* @typedef {{alive: boolean, application?: string, activeReleaseId?: string | null, proxy?: {host: string, port: number}}} ControlSocketInspection
|
|
3771
|
+
* @typedef {{alive: boolean, application?: string, activeReleaseId?: string | null, guardian?: {ownerMutation: Record<string, JsonValue> | null, version: string | null}, proxy?: {host: string, port: number}}} ControlSocketInspection
|
|
3666
3772
|
*/
|
|
3667
3773
|
|
|
3668
3774
|
/**
|
|
@@ -3737,7 +3843,7 @@ export async function inspectControlSocket(socketPath, timeoutMs = 1000) {
|
|
|
3737
3843
|
/**
|
|
3738
3844
|
* Parses a control status response line into a Rollbridge identity, if it is one.
|
|
3739
3845
|
* @param {string} line - JSON response line.
|
|
3740
|
-
* @returns {{application: string, activeReleaseId: string | null, proxy: {host: string, port: number} | undefined} | undefined} Identity, or undefined when unrecognized.
|
|
3846
|
+
* @returns {{application: string, activeReleaseId: string | null, guardian: {ownerMutation: Record<string, JsonValue> | null, version: string | null} | undefined, proxy: {host: string, port: number} | undefined} | undefined} Identity, or undefined when unrecognized.
|
|
3741
3847
|
*/
|
|
3742
3848
|
function parseControlStatus(line) {
|
|
3743
3849
|
/** @type {JsonValue} */
|
|
@@ -3754,5 +3860,8 @@ function parseControlStatus(line) {
|
|
|
3754
3860
|
|
|
3755
3861
|
const proxy = "proxy" in parsed && parsed.proxy && typeof parsed.proxy === "object" && !Array.isArray(parsed.proxy) && typeof parsed.proxy.host === "string" && typeof parsed.proxy.port === "number" ? {host: parsed.proxy.host, port: parsed.proxy.port} : undefined
|
|
3756
3862
|
|
|
3757
|
-
|
|
3863
|
+
const guardianValue = "guardian" in parsed && parsed.guardian && typeof parsed.guardian === "object" && !Array.isArray(parsed.guardian) ? parsed.guardian : undefined
|
|
3864
|
+
const guardian = guardianValue && typeof guardianValue.version === "string" || guardianValue && guardianValue.version === null ? {ownerMutation: guardianValue.ownerMutation && typeof guardianValue.ownerMutation === "object" && !Array.isArray(guardianValue.ownerMutation) ? guardianValue.ownerMutation : null, version: guardianValue.version} : undefined
|
|
3865
|
+
|
|
3866
|
+
return {activeReleaseId: typeof parsed.activeReleaseId === "string" ? parsed.activeReleaseId : null, application: parsed.application, guardian, proxy}
|
|
3758
3867
|
}
|
package/src/doctor.js
CHANGED
|
@@ -26,6 +26,13 @@ export async function runEnvironmentChecks(config) {
|
|
|
26
26
|
const socketInspection = await inspectControlSocketSafely(config.control.path)
|
|
27
27
|
|
|
28
28
|
checks.push(controlSocketCheck(config, socketInspection))
|
|
29
|
+
if (!("error" in socketInspection) && socketInspection.guardian) {
|
|
30
|
+
const {guardian} = socketInspection
|
|
31
|
+
const lease = guardian.ownerMutation
|
|
32
|
+
const leaseDetail = lease ? `, owner mutation ${String(lease.operation || "unnamed operation")} held by pid ${String(lease.pid || "unknown")}` : ", no active owner-mutation lease"
|
|
33
|
+
|
|
34
|
+
checks.push({detail: `guardian Rollbridge ${guardian.version || "unknown"}${leaseDetail}`, name: "guardian diagnostics", ok: guardian.version !== null})
|
|
35
|
+
}
|
|
29
36
|
checks.push(await controlSocketDirectoryCheck(config))
|
|
30
37
|
checks.push(await proxyPortCheck(config, socketInspection))
|
|
31
38
|
|
package/src/drain-policy.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
|
-
* @typedef {"progressing" | "wedged" | "unproven"} DrainProgress
|
|
4
|
+
* @typedef {"progressing" | "timedOut" | "wedged" | "unproven"} DrainProgress
|
|
5
5
|
* @typedef {{cpuMs: number | undefined, heldProcessGroupPgid: number | undefined, id: string, liveChildCount: number | undefined, pid: number | undefined, retirementTimedOut: boolean}} DrainProcessObservation
|
|
6
6
|
* @typedef {{atMs: number, processes: DrainProcessObservation[]}} DrainLivenessSample
|
|
7
7
|
* @typedef {{cpuDeltaMs: number | undefined, reason: string, sampleWindowMs: number, status: DrainProgress}} DrainClassification
|
|
@@ -12,26 +12,40 @@
|
|
|
12
12
|
*/
|
|
13
13
|
|
|
14
14
|
/**
|
|
15
|
-
* Classifies one draining release group as still draining or wedged.
|
|
15
|
+
* Classifies one draining release group as still draining, past its drain budget, or wedged.
|
|
16
16
|
*
|
|
17
17
|
* A group is wedged only when its drain can no longer progress. Every inconclusive reading
|
|
18
18
|
* — a live connection, a process still inside its retirement budget, an unmeasurable CPU
|
|
19
19
|
* counter, or a sample window shorter than `minSampleWindowMs` — reports `progressing` or
|
|
20
20
|
* `unproven`, never `wedged`: a legitimate multi-hour drain is valid and must not be
|
|
21
21
|
* mislabelled as a wedge.
|
|
22
|
+
*
|
|
23
|
+
* A live connection inside `drainBudgetMs` is still a legitimate drain. Past that budget the
|
|
24
|
+
* connection wait has already been given up, so the group reports `timedOut` instead of
|
|
25
|
+
* `progressing` forever — the reading that lets one held websocket outlive the activation
|
|
26
|
+
* budget and stall every later deploy. `timedOut` names the deadline; it is not an escalation
|
|
27
|
+
* and never authorises a signal.
|
|
22
28
|
* @param {object} args - Classification inputs.
|
|
23
29
|
* @param {number} args.connectionCount - Live proxy connections this generation still owns.
|
|
24
30
|
* @param {DrainLivenessSample} args.currentSample - Newest liveness observation.
|
|
31
|
+
* @param {number} args.drainBudgetMs - Configured drain budget (`proxy.drainTimeoutMs`); 0 records no budget.
|
|
32
|
+
* @param {number} args.drainElapsedMs - Time since this drain started, at classification time.
|
|
25
33
|
* @param {number} args.minSampleWindowMs - Shortest window that can prove flat CPU.
|
|
26
34
|
* @param {DrainLivenessSample | undefined} args.previousSample - Older observation, when one is in range.
|
|
27
35
|
* @returns {DrainClassification} The classification and the evidence it rests on.
|
|
28
36
|
*/
|
|
29
|
-
export function classifyDrainProgress({connectionCount, currentSample, minSampleWindowMs, previousSample}) {
|
|
37
|
+
export function classifyDrainProgress({connectionCount, currentSample, drainBudgetMs = 0, drainElapsedMs = 0, minSampleWindowMs, previousSample}) {
|
|
30
38
|
const live = currentSample.processes.filter((process) => process.pid !== undefined || process.heldProcessGroupPgid !== undefined)
|
|
31
39
|
const stillInsideBudget = live.filter((process) => !process.retirementTimedOut)
|
|
32
40
|
|
|
33
41
|
if (connectionCount > 0) {
|
|
34
|
-
|
|
42
|
+
const connections = `${connectionCount} live connection${connectionCount === 1 ? "" : "s"} still owned by the generation`
|
|
43
|
+
|
|
44
|
+
if (drainBudgetMs > 0 && drainElapsedMs > drainBudgetMs) {
|
|
45
|
+
return classify("timedOut", `drain budget ${drainBudgetMs}ms exceeded by ${drainElapsedMs - drainBudgetMs}ms with ${connections}`, currentSample, previousSample)
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
return classify("progressing", connections, currentSample, previousSample)
|
|
35
49
|
}
|
|
36
50
|
if (stillInsideBudget.length > 0) {
|
|
37
51
|
return classify("progressing", `process ${stillInsideBudget[0].id} is still inside its retirement budget`, currentSample, previousSample)
|
package/src/guardian-client.js
CHANGED
|
@@ -2,17 +2,34 @@
|
|
|
2
2
|
|
|
3
3
|
import crypto from "node:crypto"
|
|
4
4
|
import {spawn} from "node:child_process"
|
|
5
|
+
import fs from "node:fs/promises"
|
|
5
6
|
import net from "node:net"
|
|
7
|
+
import path from "node:path"
|
|
6
8
|
import {fileURLToPath} from "node:url"
|
|
7
9
|
import ManagedProcess from "./managed-process.js"
|
|
8
10
|
|
|
9
11
|
const guardianPath = fileURLToPath(new URL("./process-guardian.js", import.meta.url))
|
|
12
|
+
const MAX_UNIX_SOCKET_PATH_BYTES = 107
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Linux reserves one byte of sockaddr_un.sun_path for its terminator. Keep
|
|
16
|
+
* long task/workspace prefixes out of the pathname while retaining a stable,
|
|
17
|
+
* collision-resistant name for the configured path.
|
|
18
|
+
* @param {string} socketPath - Configured Unix socket pathname.
|
|
19
|
+
* @returns {string} Path that can be passed to net.createConnection/listen.
|
|
20
|
+
*/
|
|
21
|
+
export function usableSocketPath(socketPath) {
|
|
22
|
+
if (Buffer.byteLength(socketPath) <= MAX_UNIX_SOCKET_PATH_BYTES) return socketPath
|
|
23
|
+
|
|
24
|
+
const digest = crypto.createHash("sha256").update(socketPath).digest("hex").slice(0, 32)
|
|
25
|
+
return path.join("/tmp", `rollbridge-${digest}.sock`)
|
|
26
|
+
}
|
|
10
27
|
|
|
11
28
|
export default class GuardianClient {
|
|
12
29
|
/** @param {{pid?: number, socketPath: string, token: string}} identity - Durable guardian identity. */
|
|
13
30
|
constructor({pid, socketPath, token}) {
|
|
14
31
|
this.pid = pid
|
|
15
|
-
this.socketPath = socketPath
|
|
32
|
+
this.socketPath = usableSocketPath(socketPath)
|
|
16
33
|
this.token = token
|
|
17
34
|
this.socket = /** @type {net.Socket | undefined} */ (undefined)
|
|
18
35
|
this.buffer = ""
|
|
@@ -150,14 +167,17 @@ export default class GuardianClient {
|
|
|
150
167
|
return await response
|
|
151
168
|
}
|
|
152
169
|
|
|
153
|
-
/**
|
|
154
|
-
|
|
170
|
+
/**
|
|
171
|
+
* Stops the guardian after every owned process has stopped or reached its retirement budget.
|
|
172
|
+
* @param {{retirementTimeoutMs?: number}} [options] - Optional non-blocking retirement budget.
|
|
173
|
+
*/
|
|
174
|
+
async shutdown(options = {}) {
|
|
155
175
|
if (this.pending.size > 0) {
|
|
156
176
|
await new Promise((resolve) => {
|
|
157
177
|
this.idleWaiters.push(() => { resolve(undefined) })
|
|
158
178
|
})
|
|
159
179
|
}
|
|
160
|
-
await this.request({command: "shutdown"})
|
|
180
|
+
await this.request({command: "shutdown", options})
|
|
161
181
|
const socket = this.socket
|
|
162
182
|
|
|
163
183
|
if (!socket || socket.destroyed) throw new Error("Process guardian disconnected before shutdown acknowledgement")
|
|
@@ -186,9 +206,11 @@ export default class GuardianClient {
|
|
|
186
206
|
await this.request({command: "owner-ready", ownerPid: process.pid})
|
|
187
207
|
}
|
|
188
208
|
|
|
189
|
-
/** @returns {Promise<{daemonRecovery: number, generationReactivation?: number}>} Guardian protocol capabilities. */
|
|
209
|
+
/** @returns {Promise<{daemonRecovery: number, generationReactivation?: number, guardianVersion?: string}>} Guardian protocol capabilities. */
|
|
190
210
|
async capabilities() {
|
|
191
|
-
const capabilities = /** @type {{daemonRecovery: number, generationReactivation?: number}} */ (await this.request({command: "capabilities"}))
|
|
211
|
+
const capabilities = /** @type {{daemonRecovery: number, generationReactivation?: number, guardianVersion?: string}} */ (await this.request({command: "capabilities"}))
|
|
212
|
+
|
|
213
|
+
if (!capabilities.guardianVersion) capabilities.guardianVersion = await guardianVersionFromProcess(this.pid)
|
|
192
214
|
|
|
193
215
|
this.generationReactivation = capabilities.generationReactivation === 1
|
|
194
216
|
return capabilities
|
|
@@ -347,6 +369,11 @@ export default class GuardianClient {
|
|
|
347
369
|
await this.request({command: "report-owner-mutation-wait", mutationId, waitingFor})
|
|
348
370
|
}
|
|
349
371
|
|
|
372
|
+
/** @returns {Promise<{guardianPid: number, ownerMutation: {elapsedMs: number, leaseMs: number, mutationId: string, operation: string, pid: number | null, startedAt: string, waitingFor: string} | null, version: string}>} Guardian diagnostics. */
|
|
373
|
+
async guardianStatus() {
|
|
374
|
+
return /** @type {{guardianPid: number, ownerMutation: {elapsedMs: number, leaseMs: number, mutationId: string, operation: string, pid: number | null, startedAt: string, waitingFor: string} | null, version: string}} */ (await this.request({command: "guardian-status"}))
|
|
375
|
+
}
|
|
376
|
+
|
|
350
377
|
/** @returns {Promise<{committedReplacementId: string | null, ownerClaimed: boolean, retirementFailed?: boolean, retirementPending?: boolean, retirementReady?: boolean}>} Transaction status. */
|
|
351
378
|
async replacementStatus() {
|
|
352
379
|
return /** @type {{committedReplacementId: string | null, ownerClaimed: boolean, retirementFailed?: boolean, retirementPending?: boolean, retirementReady?: boolean}} */ (await this.request({command: "replacement-status"}))
|
|
@@ -450,6 +477,27 @@ export default class GuardianClient {
|
|
|
450
477
|
}
|
|
451
478
|
}
|
|
452
479
|
|
|
480
|
+
/**
|
|
481
|
+
* Reads the package version from a guardian's own executable path. Used when its
|
|
482
|
+
* capabilities response predates guardian version reporting, and by owner
|
|
483
|
+
* replacement, which must not send a capabilities command before a replacement
|
|
484
|
+
* transaction is prepared. An unreadable or non-Rollbridge process remains unknown.
|
|
485
|
+
* @param {number | undefined} pid - Existing guardian PID.
|
|
486
|
+
* @returns {Promise<string | undefined>} Guardian package version.
|
|
487
|
+
*/
|
|
488
|
+
export async function guardianVersionFromProcess(pid) {
|
|
489
|
+
if (!pid) return undefined
|
|
490
|
+
|
|
491
|
+
try {
|
|
492
|
+
const commandLine = (await fs.readFile(`/proc/${pid}/cmdline`, "utf8")).split("\0").find((argument) => argument.endsWith("/src/process-guardian.js"))
|
|
493
|
+
if (!commandLine) return undefined
|
|
494
|
+
const metadata = JSON.parse(await fs.readFile(path.resolve(path.dirname(commandLine), "..", "package.json"), "utf8"))
|
|
495
|
+
return metadata && typeof metadata.version === "string" ? metadata.version : undefined
|
|
496
|
+
} catch {
|
|
497
|
+
return undefined
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
|
|
453
501
|
class GuardianProcess extends ManagedProcess {
|
|
454
502
|
/** @param {{client: GuardianClient, definition: ConstructorParameters<typeof ManagedProcess>[0], key: string}} args - Remote process args. */
|
|
455
503
|
constructor({client, definition, key}) {
|
package/src/managed-process.js
CHANGED
|
@@ -22,7 +22,7 @@ const ESCALATION_KILL_CONFIRM_MS = 5000
|
|
|
22
22
|
* @typedef {import("node:child_process").ChildProcess["signalCode"]} ProcessExitSignal
|
|
23
23
|
* @typedef {{at: string, line: string, stream: "stdout" | "stderr"}} ManagedProcessLog
|
|
24
24
|
* @typedef {import("./config.js").StopTimeoutMs} StopTimeoutMs
|
|
25
|
-
* @typedef {{command: string, cwd: string | undefined, env: Record<string, string | undefined>, lifecycle: import("./config.js").LifecycleConfig, logger: (message: string, data?: Record<string, import("./json.js").JsonValue>) => void, memory: import("./config.js").MemoryConfig | undefined, outputLines: number, restart: import("./config.js").RestartConfig, restartDelayMs: number, shouldRestart: () => boolean, stopSignal: string, stopTimeoutMs: StopTimeoutMs}} ManagedProcessDefinition
|
|
25
|
+
* @typedef {{command: string, cwd: string | undefined, env: Record<string, string | undefined>, lifecycle: import("./config.js").LifecycleConfig, logger: (message: string, data?: Record<string, import("./json.js").JsonValue>) => void, memory: import("./config.js").MemoryConfig | undefined, outputLines: number, restart: import("./config.js").RestartConfig, restartDelayMs: number, shouldRestart: () => boolean, shouldStop?: () => boolean, stopSignal: string, stopTimeoutMs: StopTimeoutMs}} ManagedProcessDefinition
|
|
26
26
|
* @typedef {{liveMemberCount: number | undefined, pgid: number}} HeldProcessGroup
|
|
27
27
|
* @typedef {{children: import("./process-memory.js").ProcessGroupMember[], command: string, cwd: string | undefined, exitCode: number | null | undefined, exitSignal: ProcessExitSignal | undefined, heldProcessGroup: HeldProcessGroup | undefined, id: string, lastMemoryRestartAt: string | undefined, lastStartReason: ManagedProcessStartReason | undefined, lifecycleRole?: LifecycleRole, logs: ManagedProcessLog[], memoryRestarts: number, pid: number | undefined, restarts: number, retirementTimedOut: boolean, rssBytes: number | undefined, startedAt: string | undefined, state: ManagedProcessState, uptimeMs: number | undefined}} ManagedProcessStatus
|
|
28
28
|
*/
|
|
@@ -41,10 +41,11 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
41
41
|
* @param {import("./config.js").RestartConfig} [args.restart] - Restart policy (defaults to unlimited restarts with a constant delay).
|
|
42
42
|
* @param {number} args.restartDelayMs - Restart delay.
|
|
43
43
|
* @param {() => boolean} args.shouldRestart - Restart policy callback.
|
|
44
|
+
* @param {() => boolean} [args.shouldStop] - Whether an unexpected exit is terminal because its dependency retired.
|
|
44
45
|
* @param {string} [args.stopSignal] - Signal sent to gracefully stop the process (default "SIGTERM").
|
|
45
46
|
* @param {StopTimeoutMs} args.stopTimeoutMs - Stop timeout.
|
|
46
47
|
*/
|
|
47
|
-
constructor({command, cwd, env, id, lifecycle = {drainTimeoutMs: 0}, logger, memory, outputLines, restart = {backoffFactor: 1, maxDelayMs: 0, maxRestarts: undefined, windowMs: 0}, restartDelayMs, shouldRestart, stopSignal = "SIGTERM", stopTimeoutMs}) {
|
|
48
|
+
constructor({command, cwd, env, id, lifecycle = {drainTimeoutMs: 0}, logger, memory, outputLines, restart = {backoffFactor: 1, maxDelayMs: 0, maxRestarts: undefined, windowMs: 0}, restartDelayMs, shouldRestart, shouldStop = () => false, stopSignal = "SIGTERM", stopTimeoutMs}) {
|
|
48
49
|
super()
|
|
49
50
|
|
|
50
51
|
this.command = command
|
|
@@ -58,6 +59,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
58
59
|
this.restart = restart
|
|
59
60
|
this.restartDelayMs = restartDelayMs
|
|
60
61
|
this.shouldRestart = shouldRestart
|
|
62
|
+
this.shouldStop = shouldStop
|
|
61
63
|
this.stopSignal = stopSignal
|
|
62
64
|
this.stopTimeoutMs = stopTimeoutMs
|
|
63
65
|
this.state = /** @type {ManagedProcessState} */ ("stopped")
|
|
@@ -207,6 +209,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
207
209
|
this.restart = definition.restart
|
|
208
210
|
this.restartDelayMs = definition.restartDelayMs
|
|
209
211
|
this.shouldRestart = definition.shouldRestart
|
|
212
|
+
this.shouldStop = definition.shouldStop ?? (() => false)
|
|
210
213
|
this.stopSignal = definition.stopSignal
|
|
211
214
|
this.stopTimeoutMs = definition.stopTimeoutMs
|
|
212
215
|
}
|
|
@@ -265,12 +268,13 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
265
268
|
this.rssBytes = undefined
|
|
266
269
|
this.children = []
|
|
267
270
|
this.clearMemoryMonitor()
|
|
268
|
-
|
|
271
|
+
const terminalExit = !wasIntentional && this.shouldStop()
|
|
272
|
+
this.state = wasIntentional || terminalExit ? "stopped" : "failed"
|
|
269
273
|
|
|
270
274
|
this.logger("process exited", {code, id: this.id, signal})
|
|
271
275
|
this.emit("exit", {code, signal})
|
|
272
276
|
|
|
273
|
-
if (!wasIntentional && this.shouldRestart()) {
|
|
277
|
+
if (!wasIntentional && !terminalExit && this.shouldRestart()) {
|
|
274
278
|
this.scheduleRestart()
|
|
275
279
|
}
|
|
276
280
|
}
|
|
@@ -507,6 +511,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
507
511
|
async performStop(options) {
|
|
508
512
|
const pgid = this.child?.pid ?? this.pid
|
|
509
513
|
const exitPromise = this.exitPromise
|
|
514
|
+
const retirementDeadline = options.retirementTimeoutMs === undefined ? undefined : Date.now() + options.retirementTimeoutMs
|
|
510
515
|
await this.quiesce()
|
|
511
516
|
|
|
512
517
|
if (!pgid) {
|
|
@@ -528,7 +533,6 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
528
533
|
// 3. Stop whatever is still running, then SIGKILL if it outlasts the graceful window.
|
|
529
534
|
if (this.processGroupExists(pgid)) {
|
|
530
535
|
const timeoutMs = options.timeoutMs ?? this.stopTimeoutMs
|
|
531
|
-
const retirementDeadline = options.retirementTimeoutMs === undefined ? undefined : Date.now() + options.retirementTimeoutMs
|
|
532
536
|
const gracefulDeadline = timeoutMs === "indefinite" ? undefined : Date.now() + timeoutMs
|
|
533
537
|
// A retirement budget is the caller's decision that this process must not be waited for and
|
|
534
538
|
// must not be killed, so it caps the graceful window rather than only standing in for a
|
|
@@ -550,6 +554,11 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
550
554
|
if (!exited && retirementBound) {
|
|
551
555
|
this.retirementTimedOut = true
|
|
552
556
|
this.markGroupOutlivedStop(pgid)
|
|
557
|
+
// The tracked process has completed its supervised stop even though its process group is
|
|
558
|
+
// still draining. Keep the residue in heldProcessGroup rather than leaving the process
|
|
559
|
+
// permanently quiesced, which would make a drained generation look active to restore
|
|
560
|
+
// admission and keep the owner state from describing the actual stopped generation.
|
|
561
|
+
this.state = "stopped"
|
|
553
562
|
this.logger("process retirement wait timed out; leaving process alive", {id: this.id, pid: pgid, timeoutMs: options.retirementTimeoutMs})
|
|
554
563
|
return
|
|
555
564
|
}
|