rollbridge 0.1.72 → 0.1.74
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +63 -10
- package/TODO.md +13 -0
- package/changelog.d/20261025-guardian-re-register-dead-generation.md +13 -0
- package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +1 -0
- package/changelog.d/20261104-compose-service-units.md +21 -0
- package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +20 -0
- package/changelog.d/20261106-release-dependency-boot-budgets.md +32 -0
- package/changelog.d/20261109-path-filtered-builds.md +11 -0
- package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +26 -0
- package/changelog.d/20261111-contention-sensitive-daemon-specs.md +27 -0
- package/changelog.d/20261113-bounded-nonblocking-retirement.md +5 -0
- package/changelog.d/20261115-bound-draining-release-groups.md +62 -0
- package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +27 -0
- package/changelog.d/20261115-wedged-generation-escalation.md +44 -0
- package/changelog.d/20261116-bounded-explicit-stop-retirement.md +30 -0
- package/changelog.d/20261116-config-json-canonical-shape.md +11 -0
- package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +29 -0
- package/changelog.d/20261116-guardian-version-handoff.md +37 -0
- package/changelog.d/20261116-reclaim-dead-owner-mutation.md +7 -0
- package/changelog.d/20261117-activate-before-incumbent-retirement.md +1 -0
- package/changelog.d/20261117-activation-path-drain-deadline.md +35 -0
- package/changelog.d/20261117-stop-companion-after-retired-dependency.md +7 -0
- package/changelog.d/20261118-incumbent-retirement-authority-guard.md +1 -0
- package/changelog.d/20261118-owner-retirement-no-guardian-bound.md +19 -0
- package/changelog.d/20261118-shutdown-retirement-bound-restored.md +31 -0
- package/docs/cli.md +128 -3
- package/docs/compose-services.md +222 -0
- package/docs/config.md +177 -6
- package/docs/troubleshooting.md +97 -6
- package/docs/velocious.md +1 -1
- package/docs/workers.md +11 -0
- package/examples/tensorbuzz.com.js +3 -2
- package/package.json +1 -1
- package/src/cli.js +68 -2
- package/src/compose-unit.js +725 -0
- package/src/config.js +346 -8
- package/src/daemon.js +410 -73
- package/src/doctor.js +74 -0
- package/src/drain-policy.js +252 -0
- package/src/guardian-client.js +67 -8
- package/src/health.js +99 -8
- package/src/managed-process.js +197 -16
- package/src/process-guardian.js +212 -22
- package/src/process-memory.js +62 -2
- package/src/process-order.js +26 -0
- package/src/release-group.js +435 -37
- package/tensorbuzz.yml +20 -1
- package/test/completion.test.js +5 -2
- package/test/compose-config.test.js +160 -0
- package/test/compose-docker.test.js +297 -0
- package/test/compose-unit.test.js +415 -0
- package/test/config-validation.test.js +135 -5
- package/test/drain-policy.test.js +328 -0
- package/test/fixtures/deferred-boot-app.js +106 -0
- package/test/fixtures/gated-app.js +50 -0
- package/test/guardian-client.test.js +119 -5
- package/test/health.test.js +74 -1
- package/test/managed-process.test.js +260 -1
- package/test/owner-recovery.test.js +125 -21
- package/test/owner-replacement.test.js +175 -14
- package/test/process-memory.test.js +47 -1
- package/test/release-group.test.js +408 -2
- package/test/rollbridge.test.js +774 -72
- package/test/shutdown-completion.test.js +119 -0
- package/test/shutdown-retirement-bound.test.js +162 -0
- package/test/support/process.js +3 -1
- package/test/tensorbuzz-config.test.js +228 -0
package/src/daemon.js
CHANGED
|
@@ -9,8 +9,10 @@ import {isDeepStrictEqual} from "node:util"
|
|
|
9
9
|
import httpProxy from "http-proxy"
|
|
10
10
|
import {loadConfig} from "./config.js"
|
|
11
11
|
import {openControlSession} from "./control-client.js"
|
|
12
|
+
import {admitGenerationRelease} from "./drain-policy.js"
|
|
12
13
|
import EventLog from "./event-log.js"
|
|
13
|
-
import GuardianClient from "./guardian-client.js"
|
|
14
|
+
import GuardianClient, {guardianVersionFromProcess, usableSocketPath} from "./guardian-client.js"
|
|
15
|
+
import {childRecordIsGone} from "./managed-process.js"
|
|
14
16
|
import ReleaseGroup from "./release-group.js"
|
|
15
17
|
import {clearState, isProcessAlive, liveProcesses, readState, writeState} from "./state-store.js"
|
|
16
18
|
import {resolveGroupId, resolveUserId} from "./system-ids.js"
|
|
@@ -23,18 +25,20 @@ const STATE_PERSIST_INTERVAL_MS = 5000
|
|
|
23
25
|
* @typedef {{releaseId?: string, releasePath: string, revision?: string}} DeployArgs
|
|
24
26
|
* @typedef {{attestation?: string, releaseId: string, releasePath: string, revision: string}} BootstrapIdentity
|
|
25
27
|
* @typedef {{id: string, process: import("./managed-process.js").ManagedProcessStatus}} ProcessStatus
|
|
26
|
-
* @typedef {{disruptive: true, mode: "legacy-first-upgrade", reason: string}} OwnerTransition
|
|
28
|
+
* @typedef {{disruptive: true, mode: "guardian-version-handoff" | "legacy-first-upgrade", reason: string}} OwnerTransition
|
|
29
|
+
* @typedef {"guardian-version-drift" | "legacy-protocol"} GuardianHandoffProtocol
|
|
27
30
|
* @typedef {"candidate_ready" | "retiring_previous" | "previous_retired" | "activating_candidate" | "restoring_previous" | "retiring_failed_candidate" | "degraded_active" | "committed_pending" | "committed" | "restoring_committed"} GenerationTransitionPhase
|
|
28
31
|
* @typedef {{activationError?: string, activationLifecycle?: boolean, candidateReleaseId: string, candidateReleasePath: string, candidateRevision: string, compensationError?: string, configDigest: string, degradedIncumbent?: boolean, error?: string, journalRevision?: number, phase: GenerationTransitionPhase, previousReleaseId: string | null, startedAt: string, updatedAt: string}} GenerationTransition
|
|
29
|
-
* @typedef {{activeReleaseId: string | null, application: string, bootstrap: BootstrapIdentity | undefined, control: import("./config.js").ControlConfig, daemonPid: number, daemonRuntime: import("./daemon-runtime.js").DaemonRuntimeIdentity | undefined, generationTransition?: GenerationTransition, ownerRecovery: {configDigest: string, ready: boolean} | undefined, ownerTransition?: OwnerTransition, orphans: {id: string, pid: number, releaseId: string | null}[], proxy: {host: string, port: number | undefined, upstreamHost: string}, releaseReferences: {releaseId: string, releasePath: string}[], releases: import("./release-group.js").ReleaseStatus[], services: ProcessStatus[], singletons: ProcessStatus[]}} DaemonStatus
|
|
32
|
+
* @typedef {{activeReleaseId: string | null, application: string, bootstrap: BootstrapIdentity | undefined, control: import("./config.js").ControlConfig, daemonPid: number, daemonRuntime: import("./daemon-runtime.js").DaemonRuntimeIdentity | undefined, generationTransition?: GenerationTransition, guardian?: {guardianPid: number | null, ownerMutation: Record<string, JsonValue> | null, version: string | null}, ownerRecovery: {configDigest: string, ready: boolean} | undefined, ownerTransition?: OwnerTransition, orphans: {id: string, pid: number, releaseId: string | null}[], proxy: {host: string, port: number | undefined, upstreamHost: string}, releaseReferences: {releaseId: string, releasePath: string}[], releases: import("./release-group.js").ReleaseStatus[], services: ProcessStatus[], singletons: ProcessStatus[]}} DaemonStatus
|
|
30
33
|
* @typedef {Omit<import("./managed-process.js").ManagedProcessStatus, "logs">} ManagedProcessStatusWithoutLogs
|
|
31
34
|
* @typedef {Omit<import("./release-group.js").ReleaseStatus, "processes"> & {processes: ManagedProcessStatusWithoutLogs[]}} ReleaseStatusWithoutLogs
|
|
32
35
|
* @typedef {Omit<ProcessStatus, "process"> & {process: ManagedProcessStatusWithoutLogs}} ProcessStatusWithoutLogs
|
|
33
36
|
* @typedef {Omit<DaemonStatus, "releases" | "services" | "singletons"> & {releases: ReleaseStatusWithoutLogs[], services: ProcessStatusWithoutLogs[], singletons: ProcessStatusWithoutLogs[]}} DaemonStatusWithoutLogs
|
|
37
|
+
* @typedef {{count: number, hold: string[], processes: {heldProcessGroupPgid: number | null, pid: number | null, processId: string, state: import("./managed-process.js").ManagedProcessState}[], state: import("./release-group.js").ReleaseState}} HeldGenerationHold
|
|
34
38
|
* @typedef {{configDigest: string, format: number, guardian: {pid?: number, socketPath: string, token: string}, reconnectGraceMs: number}} OwnerRecoveryMetadata
|
|
35
39
|
* @typedef {DaemonStatus & {recovery: OwnerRecoveryMetadata, serviceReleaseIds?: Record<string, string>, singletonReleaseIds?: Record<string, string>}} OwnerRecoverySnapshot
|
|
36
40
|
* @typedef {{authority: JsonValue, config: import("./config.js").RollbridgeConfig, listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>, listenerSourceId?: string, recovery?: {command: JsonValue, reconnectGraceMs: number, startupTimeoutMs: number}, releaseConfigs?: Record<string, import("./config.js").RollbridgeConfig>, retiredListenerHandoff?: number, serviceReleaseIds?: Record<string, string>, singletonReleaseIds?: Record<string, string>, snapshot: OwnerRecoverySnapshot}} PrivateOwnerState
|
|
37
|
-
* @typedef {{boundaryCrossed: boolean, incumbentControl?: Awaited<ReturnType<typeof openControlSession>>, incumbentStartTime: string, legacyGuardian: GuardianClient, legacyInventory: {key: string, provenance: string}[], legacyPrepared?: {ownerState: JsonValue, replacementId: string}, legacySnapshot: OwnerRecoverySnapshot, prepared: {ownerState: JsonValue, replacementId: string}, recoverySnapshot: OwnerRecoverySnapshot}} LegacyOwnerBridge
|
|
41
|
+
* @typedef {{boundaryCrossed: boolean, guardianVersion?: string, incumbentControl?: Awaited<ReturnType<typeof openControlSession>>, incumbentStartTime: string, legacyGuardian: GuardianClient, legacyInventory: {key: string, provenance: string}[], legacyPrepared?: {ownerState: JsonValue, replacementId: string}, legacySnapshot: OwnerRecoverySnapshot, prepared: {ownerState: JsonValue, replacementId: string}, protocol: GuardianHandoffProtocol, recoverySnapshot: OwnerRecoverySnapshot}} LegacyOwnerBridge
|
|
38
42
|
*/
|
|
39
43
|
|
|
40
44
|
export default class RollbridgeDaemon {
|
|
@@ -146,7 +150,7 @@ export default class RollbridgeDaemon {
|
|
|
146
150
|
this.logger("release generation transition recovery failed", {error: error instanceof Error ? error.message : String(error), releaseId: transition.candidateReleaseId})
|
|
147
151
|
}
|
|
148
152
|
return {}
|
|
149
|
-
})
|
|
153
|
+
}, {waitingFor: "the retained release generations to settle"})
|
|
150
154
|
} finally {
|
|
151
155
|
this.generationTransitionRecovery = false
|
|
152
156
|
}
|
|
@@ -203,7 +207,7 @@ export default class RollbridgeDaemon {
|
|
|
203
207
|
if (!snapshot || snapshot.recovery?.guardian || !exactBootstrap) throw new Error("Retired owner reset requires an exact committed bootstrap with missing guardian identity")
|
|
204
208
|
const control = await inspectControlSocket(this.config.control.path)
|
|
205
209
|
if (control.alive) throw new Error(`Retired owner reset refuses responsive control owner at ${this.config.control.path}`)
|
|
206
|
-
this.guardianIdentity = {socketPath: `${this.statePath}.guardian.sock
|
|
210
|
+
this.guardianIdentity = {socketPath: usableSocketPath(`${this.statePath}.guardian.sock`), token: crypto.randomBytes(32).toString("hex")}
|
|
207
211
|
this.guardian = new GuardianClient(this.guardianIdentity)
|
|
208
212
|
await this.guardian.launch()
|
|
209
213
|
this.guardianIdentity.pid = this.guardian.pid
|
|
@@ -221,10 +225,12 @@ export default class RollbridgeDaemon {
|
|
|
221
225
|
|
|
222
226
|
if (snapshot && !recovery) throw new Error(`Owner recovery state ${this.statePath} is missing durable guardian identity; refusing to overwrite it.`)
|
|
223
227
|
|
|
224
|
-
const guardianIdentity = recovery?.guardian
|
|
225
|
-
socketPath:
|
|
226
|
-
|
|
227
|
-
|
|
228
|
+
const guardianIdentity = recovery?.guardian
|
|
229
|
+
? {...recovery.guardian, socketPath: usableSocketPath(recovery.guardian.socketPath)}
|
|
230
|
+
: {
|
|
231
|
+
socketPath: usableSocketPath(`${this.statePath}.guardian.sock`),
|
|
232
|
+
token: crypto.randomBytes(32).toString("hex")
|
|
233
|
+
}
|
|
228
234
|
this.guardianIdentity = guardianIdentity
|
|
229
235
|
this.guardian = new GuardianClient(guardianIdentity)
|
|
230
236
|
if (recovery) {
|
|
@@ -500,15 +506,36 @@ export default class RollbridgeDaemon {
|
|
|
500
506
|
/**
|
|
501
507
|
* Prepares from private guardian state, starts listeners, then asks the committed owner
|
|
502
508
|
* to atomically fence itself and transfer authority.
|
|
509
|
+
* @returns {Promise<void>} Resolves after the replacement is committed.
|
|
503
510
|
*/
|
|
504
511
|
async replaceIncompatibleOwner() {
|
|
512
|
+
const maxAttempts = 3
|
|
513
|
+
let lastError
|
|
514
|
+
|
|
515
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
|
516
|
+
try {
|
|
517
|
+
return await this.replaceIncompatibleOwnerAttempt()
|
|
518
|
+
} catch (error) {
|
|
519
|
+
lastError = error
|
|
520
|
+
const diagnostic = error instanceof Error ? error.message : String(error)
|
|
521
|
+
|
|
522
|
+
if (!/^Guardian owner state changed after prepare \(prepared revision \d+, current revision \d+\); abort the stale candidate before preparing again$/.test(diagnostic) || attempt === maxAttempts) {
|
|
523
|
+
throw error
|
|
524
|
+
}
|
|
525
|
+
this.logger("owner replacement candidate became stale; re-preparing", {attempt, maxAttempts, error: diagnostic})
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
throw lastError
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
async replaceIncompatibleOwnerAttempt() {
|
|
505
532
|
if (!this.statePath || !this.config.ownerRecovery) throw new Error("Atomic owner replacement requires ownerRecovery and the same statePath transaction anchor")
|
|
506
533
|
this.controlCommandsReady = false
|
|
507
534
|
const state = await readState(this.statePath)
|
|
508
535
|
const persisted = state && typeof state === "object" && !Array.isArray(state) ? /** @type {OwnerRecoverySnapshot} */ (state) : undefined
|
|
509
536
|
|
|
510
537
|
if (!persisted?.recovery?.guardian) throw new Error(`No committed durable owner transaction was found at ${this.statePath}`)
|
|
511
|
-
this.guardianIdentity = persisted.recovery.guardian
|
|
538
|
+
this.guardianIdentity = {...persisted.recovery.guardian, socketPath: usableSocketPath(persisted.recovery.guardian.socketPath)}
|
|
512
539
|
this.guardian = new GuardianClient(this.guardianIdentity)
|
|
513
540
|
await this.guardian.connect()
|
|
514
541
|
this.watchOwnerReplacementEvents()
|
|
@@ -516,9 +543,16 @@ export default class RollbridgeDaemon {
|
|
|
516
543
|
configDigest: persisted.recovery.configDigest,
|
|
517
544
|
runtime: persisted.daemonRuntime ? {...persisted.daemonRuntime} : null
|
|
518
545
|
}
|
|
519
|
-
const
|
|
546
|
+
const guardianVersion = await this.retainedGuardianVersion()
|
|
547
|
+
const versionDrift = Boolean(this.runtime?.version && guardianVersion && guardianVersion !== this.runtime.version)
|
|
548
|
+
const replacementProtocol = versionDrift ? "legacy" : await this.guardian.ownerReplacementProtocol()
|
|
520
549
|
const legacyBridge = replacementProtocol === "legacy"
|
|
521
|
-
? await this.prepareLegacyOwnerReplacement({
|
|
550
|
+
? await this.prepareLegacyOwnerReplacement({
|
|
551
|
+
guardianVersion,
|
|
552
|
+
persisted,
|
|
553
|
+
persistedAuthority,
|
|
554
|
+
protocol: versionDrift ? "guardian-version-drift" : "legacy-protocol"
|
|
555
|
+
})
|
|
522
556
|
: undefined
|
|
523
557
|
const prepared = legacyBridge?.prepared ?? await this.guardian.prepareOwnerReplacement(persistedAuthority, this.ownerAuthority())
|
|
524
558
|
let preparedStatus
|
|
@@ -764,6 +798,33 @@ export default class RollbridgeDaemon {
|
|
|
764
798
|
}
|
|
765
799
|
}
|
|
766
800
|
|
|
801
|
+
/**
|
|
802
|
+
* Reads the retained guardian's package version without deciding the replacement protocol.
|
|
803
|
+
*
|
|
804
|
+
* The version is read from the guardian's own executable rather than its capabilities
|
|
805
|
+
* command: a pre-replacement guardian only answers that command once a replacement
|
|
806
|
+
* transaction is prepared, so asking here would move the probe ahead of the transaction
|
|
807
|
+
* every capability-gated guardian depends on. A guardian whose version cannot be read
|
|
808
|
+
* stays on the protocol classification path.
|
|
809
|
+
* @returns {Promise<string | undefined>} Retained guardian package version.
|
|
810
|
+
*/
|
|
811
|
+
async retainedGuardianVersion() {
|
|
812
|
+
if (!this.guardian) throw new Error("Guardian version reconciliation requires the durable process guardian")
|
|
813
|
+
return await guardianVersionFromProcess(this.guardian.pid)
|
|
814
|
+
}
|
|
815
|
+
|
|
816
|
+
/**
|
|
817
|
+
* Names the last-resort refusal for a guardian this daemon cannot hand off automatically.
|
|
818
|
+
* @param {string | undefined} guardianVersion - Retained guardian package version.
|
|
819
|
+
* @param {string} cause - Measured reason the automatic handoff cannot run.
|
|
820
|
+
* @returns {Error} Refusal naming both versions and the exact repair command.
|
|
821
|
+
*/
|
|
822
|
+
guardianVersionHandoffRefusal(guardianVersion, cause) {
|
|
823
|
+
const configArgument = this.configPath ? ` --config ${this.configPath}` : ""
|
|
824
|
+
|
|
825
|
+
return new Error(`The persistent Rollbridge guardian is version ${guardianVersion ?? "unknown"} while this daemon runs rollbridge ${this.runtime?.version ?? "unknown"}; ${cause}. Run "rollbridge shutdown${configArgument}" and then "rollbridge ensure-daemon${configArgument}" to replace the supervisor with this Rollbridge installation.`)
|
|
826
|
+
}
|
|
827
|
+
|
|
767
828
|
/** Requires the persistent guardian protocol which can restart an accepted daemon. */
|
|
768
829
|
async assertGuardianDaemonRecoveryCapability() {
|
|
769
830
|
if (!this.guardian) throw new Error("Owner recovery capability check requires the durable process guardian")
|
|
@@ -772,10 +833,18 @@ export default class RollbridgeDaemon {
|
|
|
772
833
|
try {
|
|
773
834
|
capabilities = await this.guardian.capabilities()
|
|
774
835
|
} catch (error) {
|
|
775
|
-
if (!(error instanceof Error) || !
|
|
836
|
+
if (!(error instanceof Error) || !isGuardianDaemonRecoveryDiagnostic(error.message)) throw error
|
|
776
837
|
throw new Error("The persistent Rollbridge guardian predates daemon recovery. Perform one explicit clean Rollbridge shutdown and restart before retrying this package upgrade.", {cause: error})
|
|
777
838
|
}
|
|
778
839
|
if (capabilities.daemonRecovery !== 1) throw new Error("The persistent Rollbridge guardian does not support the required daemon recovery protocol")
|
|
840
|
+
const clientVersion = this.runtime?.version
|
|
841
|
+
const guardianVersion = capabilities.guardianVersion
|
|
842
|
+
|
|
843
|
+
if (!clientVersion) return
|
|
844
|
+
if (!guardianVersion) throw this.guardianVersionHandoffRefusal(guardianVersion, "its version cannot be attested, so no handoff can be authorised")
|
|
845
|
+
if (guardianVersion !== clientVersion) {
|
|
846
|
+
throw this.guardianVersionHandoffRefusal(guardianVersion, "this startup path has no incumbent daemon to hand guardian ownership from")
|
|
847
|
+
}
|
|
779
848
|
}
|
|
780
849
|
|
|
781
850
|
/**
|
|
@@ -896,13 +965,15 @@ export default class RollbridgeDaemon {
|
|
|
896
965
|
}
|
|
897
966
|
|
|
898
967
|
/**
|
|
899
|
-
* Authenticates and prepares the
|
|
900
|
-
*
|
|
968
|
+
* Authenticates and prepares the disruptive guardian-upgrade bridge, for a guardian that predates
|
|
969
|
+
* the replacement protocol or whose package version drifted from this daemon's runtime.
|
|
970
|
+
* @param {{guardianVersion?: string, persisted: OwnerRecoverySnapshot, persistedAuthority: {configDigest: string, runtime: import("./daemon-runtime.js").DaemonRuntimeIdentity | null}, protocol: GuardianHandoffProtocol}} options - Bridge evidence and why the bridge is required.
|
|
901
971
|
* @returns {Promise<LegacyOwnerBridge>} Prepared bridge.
|
|
902
972
|
*/
|
|
903
|
-
async prepareLegacyOwnerReplacement({persisted, persistedAuthority}) {
|
|
973
|
+
async prepareLegacyOwnerReplacement({guardianVersion, persisted, persistedAuthority, protocol}) {
|
|
904
974
|
const legacyProcessKey = ownerSnapshotProcessKeys(persisted, persisted.singletonReleaseIds)[0]
|
|
905
975
|
const legacyGuardian = this.guardian
|
|
976
|
+
const versionHandoff = protocol === "guardian-version-drift"
|
|
906
977
|
let legacyPrepared
|
|
907
978
|
|
|
908
979
|
if (!legacyProcessKey) throw new Error("Legacy disruptive owner replacement requires an exact guardian-owned process registration in durable state")
|
|
@@ -935,12 +1006,23 @@ export default class RollbridgeDaemon {
|
|
|
935
1006
|
|
|
936
1007
|
try {
|
|
937
1008
|
if (persisted.recovery.configDigest !== this.ownerRecoveryConfigDigest()) {
|
|
1009
|
+
if (versionHandoff) throw this.guardianVersionHandoffRefusal(guardianVersion, "the handoff keeps the incumbent config identity unchanged and this deploy changes config authority")
|
|
938
1010
|
throw new Error("The one-time legacy guardian bridge requires the incumbent config identity unchanged; retry the config change after the bridge establishes split-3 authority")
|
|
939
1011
|
}
|
|
940
|
-
if (!this.legacyIncumbentPid)
|
|
1012
|
+
if (!this.legacyIncumbentPid) {
|
|
1013
|
+
if (versionHandoff) throw this.guardianVersionHandoffRefusal(guardianVersion, "ensure-daemon reported no incumbent daemon to hand guardian ownership from")
|
|
1014
|
+
throw new Error("The authenticated legacy guardian requires an exact incumbent PID from ensure-daemon for the disruptive bridge")
|
|
1015
|
+
}
|
|
941
1016
|
if (!this.guardianIdentity?.pid) throw new Error("The authenticated legacy guardian state is missing its exact guardian PID")
|
|
942
1017
|
await verifyLegacyGuardianProcess(this.guardianIdentity.pid, this.guardianIdentity.socketPath)
|
|
943
|
-
|
|
1018
|
+
let incumbentStartTime
|
|
1019
|
+
|
|
1020
|
+
try {
|
|
1021
|
+
incumbentStartTime = await verifyLegacyDaemonProcess(this.legacyIncumbentPid, this.configPath, persisted.control.path)
|
|
1022
|
+
} catch (error) {
|
|
1023
|
+
if (!versionHandoff) throw error
|
|
1024
|
+
throw this.guardianVersionHandoffRefusal(guardianVersion, `the incumbent daemon PID ${this.legacyIncumbentPid} is not the attested owner: ${error instanceof Error ? error.message : String(error)}`)
|
|
1025
|
+
}
|
|
944
1026
|
if (legacyPrepared) assertLegacyPrivateOwnerState(legacyPrepared.ownerState, persisted, persistedAuthority)
|
|
945
1027
|
try {
|
|
946
1028
|
incumbentControl = await openControlSession(persisted.control.path)
|
|
@@ -967,7 +1049,9 @@ export default class RollbridgeDaemon {
|
|
|
967
1049
|
|
|
968
1050
|
assertLegacyGuardianInventoryMembership(legacyInventory, /** @type {PrivateOwnerState} */ (ownerState))
|
|
969
1051
|
const upgradedIdentity = /** @type {{pid?: number, socketPath: string, token: string}} */ ({
|
|
970
|
-
socketPath:
|
|
1052
|
+
socketPath: usableSocketPath(versionHandoff
|
|
1053
|
+
? `${this.statePath}.guardian-${this.runtime?.version}.sock`
|
|
1054
|
+
: `${this.statePath}.split3-guardian.sock`),
|
|
971
1055
|
token: crypto.randomBytes(32).toString("hex")
|
|
972
1056
|
})
|
|
973
1057
|
|
|
@@ -982,12 +1066,15 @@ export default class RollbridgeDaemon {
|
|
|
982
1066
|
recovery: {...persisted.recovery, guardian: upgradedIdentity}
|
|
983
1067
|
})
|
|
984
1068
|
|
|
985
|
-
this.logger("
|
|
1069
|
+
this.logger("guardian upgrade bridge prepared", {
|
|
986
1070
|
guardianPid: upgradedIdentity.pid ?? null,
|
|
1071
|
+
guardianVersion: guardianVersion ?? "unknown",
|
|
987
1072
|
incumbentPid: this.legacyIncumbentPid,
|
|
988
|
-
|
|
1073
|
+
protocol,
|
|
1074
|
+
replacementId: prepared.replacementId,
|
|
1075
|
+
runtimeVersion: this.runtime?.version ?? "unknown"
|
|
989
1076
|
})
|
|
990
|
-
return {boundaryCrossed: false, incumbentControl, incumbentStartTime, legacyGuardian, legacyInventory, legacyPrepared, legacySnapshot: persisted, prepared, recoverySnapshot}
|
|
1077
|
+
return {boundaryCrossed: false, guardianVersion, incumbentControl, incumbentStartTime, legacyGuardian, legacyInventory, legacyPrepared, legacySnapshot: persisted, prepared, protocol, recoverySnapshot}
|
|
991
1078
|
} catch (upgradeError) {
|
|
992
1079
|
incumbentControl?.close()
|
|
993
1080
|
let cleanupError
|
|
@@ -1067,10 +1154,20 @@ export default class RollbridgeDaemon {
|
|
|
1067
1154
|
|
|
1068
1155
|
if (staged.committed) throw new Error("Legacy guardian committed replacement before the authenticated disruptive boundary")
|
|
1069
1156
|
}
|
|
1070
|
-
|
|
1157
|
+
const boundaryReason = bridge.protocol === "guardian-version-drift"
|
|
1158
|
+
? `retained guardian ${bridge.guardianVersion ?? "unknown"} predates this daemon runtime ${this.runtime?.version ?? "unknown"}`
|
|
1159
|
+
: "retained guardian and daemon lacked atomic replacement protocol"
|
|
1160
|
+
|
|
1161
|
+
this.logger("guardian upgrade bridge crossed", {
|
|
1071
1162
|
disruptive: true,
|
|
1163
|
+
guardianVersion: bridge.guardianVersion ?? "unknown",
|
|
1072
1164
|
incumbentPid: this.legacyIncumbentPid,
|
|
1073
|
-
|
|
1165
|
+
protocol: bridge.protocol,
|
|
1166
|
+
reason: boundaryReason,
|
|
1167
|
+
// The retained guardian keeps its own children; the boundary retires only the incumbent daemon.
|
|
1168
|
+
retainedGuardianPid: bridge.legacyGuardian.pid ?? null,
|
|
1169
|
+
retainedGuardianProcesses: bridge.legacyInventory.length,
|
|
1170
|
+
runtimeVersion: this.runtime?.version ?? "unknown"
|
|
1074
1171
|
})
|
|
1075
1172
|
await this.guardian.beginLegacyOwnerClaim(
|
|
1076
1173
|
bridge.prepared.replacementId,
|
|
@@ -1086,8 +1183,8 @@ export default class RollbridgeDaemon {
|
|
|
1086
1183
|
await this.guardian.completeLegacyOwnerClaim(bridge.prepared.replacementId)
|
|
1087
1184
|
this.ownerTransition = /** @type {OwnerTransition} */ ({
|
|
1088
1185
|
disruptive: true,
|
|
1089
|
-
mode: "legacy-first-upgrade",
|
|
1090
|
-
reason:
|
|
1186
|
+
mode: bridge.protocol === "guardian-version-drift" ? "guardian-version-handoff" : "legacy-first-upgrade",
|
|
1187
|
+
reason: boundaryReason
|
|
1091
1188
|
})
|
|
1092
1189
|
}
|
|
1093
1190
|
|
|
@@ -1389,7 +1486,7 @@ export default class RollbridgeDaemon {
|
|
|
1389
1486
|
releaseId: stringOrUndefined(data.releaseId),
|
|
1390
1487
|
releasePath: requiredString(data.releasePath, "releasePath"),
|
|
1391
1488
|
revision: stringOrUndefined(data.revision)
|
|
1392
|
-
}))
|
|
1489
|
+
}), {waitingFor: "candidate activation and the retired generation handoff"})
|
|
1393
1490
|
}
|
|
1394
1491
|
|
|
1395
1492
|
if (commandName === "recover-generation-transition") {
|
|
@@ -1399,11 +1496,22 @@ export default class RollbridgeDaemon {
|
|
|
1399
1496
|
releaseId: requiredString(data.releaseId, "releaseId"),
|
|
1400
1497
|
releasePath: requiredString(data.releasePath, "releasePath"),
|
|
1401
1498
|
revision: requiredString(data.revision, "revision")
|
|
1402
|
-
}))
|
|
1499
|
+
}), {waitingFor: "the retained release generations to settle"})
|
|
1403
1500
|
}
|
|
1404
1501
|
|
|
1405
1502
|
if (commandName === "status") {
|
|
1406
|
-
|
|
1503
|
+
const status = this.status({includeLogs: statusIncludeLogs(data.includeLogs)})
|
|
1504
|
+
let guardian
|
|
1505
|
+
|
|
1506
|
+
try {
|
|
1507
|
+
const diagnostics = await this.guardian?.guardianStatus()
|
|
1508
|
+
|
|
1509
|
+
guardian = diagnostics ? {guardianPid: diagnostics.guardianPid, ownerMutation: diagnostics.ownerMutation, version: diagnostics.version} : {guardianPid: null, ownerMutation: null, version: null}
|
|
1510
|
+
} catch (error) {
|
|
1511
|
+
guardian = {error: error instanceof Error ? error.message : String(error), guardianPid: this.guardianIdentity?.pid ?? null, ownerMutation: null, version: null}
|
|
1512
|
+
}
|
|
1513
|
+
|
|
1514
|
+
return {...status, guardian}
|
|
1407
1515
|
}
|
|
1408
1516
|
|
|
1409
1517
|
if (commandName === "events") {
|
|
@@ -1414,18 +1522,18 @@ export default class RollbridgeDaemon {
|
|
|
1414
1522
|
return await this.executeOwnerMutation("stop", async () => {
|
|
1415
1523
|
await this.stopRelease(stringOrUndefined(data.releaseId))
|
|
1416
1524
|
return this.status()
|
|
1417
|
-
})
|
|
1525
|
+
}, {waitingFor: "the release's managed processes to stop"})
|
|
1418
1526
|
}
|
|
1419
1527
|
|
|
1420
1528
|
if (commandName === "restart") {
|
|
1421
1529
|
return await this.executeOwnerMutation("restart", async () => await this.restartProcesses({
|
|
1422
1530
|
policy: stringOrUndefined(data.policy),
|
|
1423
1531
|
processId: stringOrUndefined(data.processId)
|
|
1424
|
-
}))
|
|
1532
|
+
}), {waitingFor: "the managed process to stop and start again"})
|
|
1425
1533
|
}
|
|
1426
1534
|
|
|
1427
1535
|
if (commandName === "rollback") {
|
|
1428
|
-
return await this.executeOwnerMutation("rollback", async () => await this.rollback({releaseId: stringOrUndefined(data.releaseId)}))
|
|
1536
|
+
return await this.executeOwnerMutation("rollback", async () => await this.rollback({releaseId: stringOrUndefined(data.releaseId)}), {waitingFor: "the previous release to activate"})
|
|
1429
1537
|
}
|
|
1430
1538
|
|
|
1431
1539
|
if (commandName === "shutdown") {
|
|
@@ -1489,11 +1597,13 @@ export default class RollbridgeDaemon {
|
|
|
1489
1597
|
* Runs one owner mutation under the guardian's prepare/commit exclusion fence.
|
|
1490
1598
|
* @param {string} operation - Mutation diagnostic name.
|
|
1491
1599
|
* @param {() => Promise<Record<string, JsonValue>>} callback - Mutating operation.
|
|
1600
|
+
* @param {{waitingFor?: string}} [options] - What this mutation blocks on, so a refused
|
|
1601
|
+
* later mutation names the real blocker instead of just "in progress".
|
|
1492
1602
|
* @returns {Promise<Record<string, JsonValue>>} Operation result.
|
|
1493
1603
|
*/
|
|
1494
|
-
async executeOwnerMutation(operation, callback) {
|
|
1604
|
+
async executeOwnerMutation(operation, callback, {waitingFor} = {}) {
|
|
1495
1605
|
if (!this.guardian) return await callback()
|
|
1496
|
-
const mutationId = await this.guardian.beginOwnerMutation(operation)
|
|
1606
|
+
const mutationId = await this.guardian.beginOwnerMutation(operation, {ownerPid: process.pid, waitingFor})
|
|
1497
1607
|
let result
|
|
1498
1608
|
let operationError
|
|
1499
1609
|
|
|
@@ -1629,6 +1739,46 @@ export default class RollbridgeDaemon {
|
|
|
1629
1739
|
await this.updateGenerationTransition("restoring_committed")
|
|
1630
1740
|
return await this.resumeGenerationTransition()
|
|
1631
1741
|
}
|
|
1742
|
+
await this.escalateWedgedGenerations(nextConfig, newReleaseId)
|
|
1743
|
+
// A release id that is still retained by a generation holding live work cannot be started
|
|
1744
|
+
// a second time: the candidate's processes re-register the guardian keys that the retained
|
|
1745
|
+
// generation's own processes still hold, so the deploy aborts partway through `start()`
|
|
1746
|
+
// with a bare `Guardian provenance mismatch` — after the caller has committed to this
|
|
1747
|
+
// revision and, for a restore, after the deployment pointer has already moved. Refusing
|
|
1748
|
+
// from the retained record before any process starts keeps the incumbent serving, names the
|
|
1749
|
+
// release the guardian reports as active, and leaves the retained generation intact for
|
|
1750
|
+
// `rollbridge status` to reconcile against. Journaled transition resume above is exempt:
|
|
1751
|
+
// it re-drives the transaction that already owns this release id.
|
|
1752
|
+
//
|
|
1753
|
+
// The escalation above runs first on purpose: it is what clears the residue that made this
|
|
1754
|
+
// guard refuse the measured 2026-11-15 and 2026-11-16 deploys. A wedge that escalation can
|
|
1755
|
+
// prove is kill-safe stops holding its keys and the handoff proceeds; a generation that
|
|
1756
|
+
// still holds live work is not killed and is named here instead of aborting mid-`start()`.
|
|
1757
|
+
const heldGeneration = this.retainedGenerationHoldingLiveWork(newReleaseId)
|
|
1758
|
+
|
|
1759
|
+
if (heldGeneration) {
|
|
1760
|
+
const activeReleaseId = this.activeRelease?.releaseId ?? null
|
|
1761
|
+
|
|
1762
|
+
this.logger("deploy refused: retained generation still holds live processes", {
|
|
1763
|
+
activeReleaseId,
|
|
1764
|
+
heldProcesses: heldGeneration.processes,
|
|
1765
|
+
releaseId: newReleaseId,
|
|
1766
|
+
retainedState: heldGeneration.state
|
|
1767
|
+
})
|
|
1768
|
+
|
|
1769
|
+
throw new Error(`Deploy refused before activation: release "${newReleaseId}" is retained in state ${heldGeneration.state} while ${heldGeneration.count} process${heldGeneration.count === 1 ? "" : "es"} of that generation still hold live work (${heldGeneration.hold.join("; ")}). Activating that release id again would re-register those processes under a new provenance and abort with a guardian provenance mismatch. The guardian's active release is ${activeReleaseId ? `"${activeReleaseId}"` : "none"}; deploy a new release id or reconcile the deployment pointer with \`rollbridge status\`.`)
|
|
1770
|
+
}
|
|
1771
|
+
const admission = this.evaluateGenerationAdmission(newReleaseId, nextConfig)
|
|
1772
|
+
|
|
1773
|
+
if (!admission.allowed) {
|
|
1774
|
+
this.logger("deploy refused by draining-generation bound", {
|
|
1775
|
+
candidateReleaseId: newReleaseId,
|
|
1776
|
+
draining: admission.draining.map((group) => ({releaseId: group.releaseId, status: group.status})),
|
|
1777
|
+
limit: nextConfig.releaseRetention.maxConcurrentDrainingReleaseGroups,
|
|
1778
|
+
wouldBeDrainingCount: admission.wouldBeDrainingCount
|
|
1779
|
+
})
|
|
1780
|
+
throw new Error(/** @type {string} */ (admission.reason))
|
|
1781
|
+
}
|
|
1632
1782
|
const release = new ReleaseGroup({
|
|
1633
1783
|
config: nextConfig,
|
|
1634
1784
|
logger: this.logger,
|
|
@@ -1685,6 +1835,79 @@ export default class RollbridgeDaemon {
|
|
|
1685
1835
|
return await this.resumeGenerationTransition()
|
|
1686
1836
|
}
|
|
1687
1837
|
|
|
1838
|
+
/**
|
|
1839
|
+
* Applies the draining-generation bound before a handoff.
|
|
1840
|
+
*
|
|
1841
|
+
* Without this, every deploy retires one more generation and the backlog grows
|
|
1842
|
+
* monotonically whenever a drain outlasts the deploy cadence; a consumer that checks the
|
|
1843
|
+
* count after activation reports the growth it cannot undo. Refusing here names the
|
|
1844
|
+
* draining groups and whether each is still progressing or wedged, before any process of
|
|
1845
|
+
* the candidate is started.
|
|
1846
|
+
* @param {string} candidateReleaseId - Release the handoff would activate.
|
|
1847
|
+
* @param {import("./config.js").RollbridgeConfig} config - Config governing the handoff.
|
|
1848
|
+
* @returns {import("./drain-policy.js").GenerationAdmission} Admission decision and the groups behind it.
|
|
1849
|
+
*/
|
|
1850
|
+
evaluateGenerationAdmission(candidateReleaseId, config) {
|
|
1851
|
+
const draining = [...this.releases.values()]
|
|
1852
|
+
.filter((release) => release.state === "draining")
|
|
1853
|
+
.map((release) => {
|
|
1854
|
+
const drain = release.drainStatus()
|
|
1855
|
+
|
|
1856
|
+
return {reason: drain?.reason ?? "no liveness observation recorded", releaseId: release.releaseId, status: drain?.status ?? "unproven"}
|
|
1857
|
+
})
|
|
1858
|
+
|
|
1859
|
+
return admitGenerationRelease({
|
|
1860
|
+
candidateReleaseId,
|
|
1861
|
+
draining,
|
|
1862
|
+
incumbentWillDrain: this.activeRelease !== undefined,
|
|
1863
|
+
maxConcurrentDrainingReleaseGroups: config.releaseRetention.maxConcurrentDrainingReleaseGroups
|
|
1864
|
+
})
|
|
1865
|
+
}
|
|
1866
|
+
|
|
1867
|
+
/**
|
|
1868
|
+
* Reports whether a retained release group still holds live work that a re-activation would collide with.
|
|
1869
|
+
*
|
|
1870
|
+
* `stopped` is not the same claim as `drained`: the guardian's provenance fence only lets a
|
|
1871
|
+
* process key be re-registered once the record that holds it is gone. A generation recorded
|
|
1872
|
+
* as drained while its process group is still alive — the shape an indefinite worker drain
|
|
1873
|
+
* produces — still holds its keys, so its release id is not available for a restore.
|
|
1874
|
+
* @param {string} releaseId - Release id a handoff would activate.
|
|
1875
|
+
* @returns {HeldGenerationHold | undefined} The live hold, or undefined when the id is free.
|
|
1876
|
+
*/
|
|
1877
|
+
retainedGenerationHoldingLiveWork(releaseId) {
|
|
1878
|
+
const retained = this.releases.get(releaseId)
|
|
1879
|
+
|
|
1880
|
+
if (!retained || retained === this.activeRelease) return undefined
|
|
1881
|
+
|
|
1882
|
+
return heldGenerationHold(retained)
|
|
1883
|
+
}
|
|
1884
|
+
|
|
1885
|
+
/**
|
|
1886
|
+
* Gives every retiring generation the escalation decision its own retirement left open,
|
|
1887
|
+
* before the handoff is judged.
|
|
1888
|
+
*
|
|
1889
|
+
* A deploy is the event that makes a wedged generation expensive: until something tries to
|
|
1890
|
+
* activate a successor, a retired release holding a live process group costs nothing
|
|
1891
|
+
* observable. Acting here means the observation that proves the wedge and the action on it
|
|
1892
|
+
* come from the same read the admission bound already performs, so no sampler has to run
|
|
1893
|
+
* alongside the daemon. Generations that already recorded `release drained` are included:
|
|
1894
|
+
* their retirement finished, and the process group they left behind is the residue that
|
|
1895
|
+
* blocked the measured 2026-11-15 deploys.
|
|
1896
|
+
* @param {import("./config.js").RollbridgeConfig} config - Config governing the handoff.
|
|
1897
|
+
* @param {string} candidateReleaseId - Release the handoff would activate.
|
|
1898
|
+
* @returns {Promise<void>} Resolves once every retiring generation has been decided.
|
|
1899
|
+
*/
|
|
1900
|
+
async escalateWedgedGenerations(config, candidateReleaseId) {
|
|
1901
|
+
if (!config.releaseRetention.escalateWedgedGenerations) return
|
|
1902
|
+
|
|
1903
|
+
for (const release of this.releases.values()) {
|
|
1904
|
+
if (release.releaseId === candidateReleaseId) continue
|
|
1905
|
+
if (release.state !== "draining" && release.state !== "stopped" && release.state !== "failed") continue
|
|
1906
|
+
|
|
1907
|
+
await release.escalateWedgedGeneration({reason: `deploy of ${candidateReleaseId}`})
|
|
1908
|
+
}
|
|
1909
|
+
}
|
|
1910
|
+
|
|
1688
1911
|
/**
|
|
1689
1912
|
* Continues one exact durable generation transition without an internal retry loop.
|
|
1690
1913
|
* @returns {Promise<Record<string, JsonValue>>} Deploy result after commit.
|
|
@@ -1706,10 +1929,9 @@ export default class RollbridgeDaemon {
|
|
|
1706
1929
|
let retirementFailure = activationLifecycle ? undefined : previousRelease?.retirementError
|
|
1707
1930
|
|
|
1708
1931
|
if (transition.phase === "candidate_ready") {
|
|
1709
|
-
//
|
|
1710
|
-
//
|
|
1711
|
-
//
|
|
1712
|
-
// carries traffic (STOE-2269).
|
|
1932
|
+
// The candidate has already started and passed its health gates. Keep the
|
|
1933
|
+
// incumbent authoritative until the candidate is activated and traffic is
|
|
1934
|
+
// switched; retirement is deliberately launched from committed_pending.
|
|
1713
1935
|
await this.updateGenerationTransition("activating_candidate")
|
|
1714
1936
|
}
|
|
1715
1937
|
|
|
@@ -1784,6 +2006,22 @@ export default class RollbridgeDaemon {
|
|
|
1784
2006
|
|
|
1785
2007
|
if (transition.phase === "committed_pending") {
|
|
1786
2008
|
await this.refreshServiceDefinitions(release)
|
|
2009
|
+
if (previousRelease?.state === "active") {
|
|
2010
|
+
if (this.activeRelease !== release) {
|
|
2011
|
+
throw new Error(`Cannot retire incumbent release ${previousRelease.releaseId} before candidate release ${release.releaseId} is authoritative.`)
|
|
2012
|
+
}
|
|
2013
|
+
const retirementConfig = previousRelease.preserveConfigOnRetirement ? previousRelease.config : release.config
|
|
2014
|
+
|
|
2015
|
+
// Traffic is already on the healthy candidate. Retirement must not hold
|
|
2016
|
+
// the deploy mutation or its owner fence; its own bounded drain reports
|
|
2017
|
+
// failures and escalates any process that misses the retirement budget.
|
|
2018
|
+
void previousRelease.beginRetirement(retirementConfig)
|
|
2019
|
+
.catch((error) => {
|
|
2020
|
+
retirementFailure = previousRelease.retirementError ?? (error instanceof Error ? error.message : String(error))
|
|
2021
|
+
this.logger("release retirement quiescence failed", {error: retirementFailure, releaseId: previousRelease.releaseId})
|
|
2022
|
+
})
|
|
2023
|
+
.then(() => this.drainAndPrune(previousRelease, retirementConfig))
|
|
2024
|
+
}
|
|
1787
2025
|
await this.checkpointGenerationTransition()
|
|
1788
2026
|
try {
|
|
1789
2027
|
await this.replaceSingletons(release)
|
|
@@ -1791,29 +2029,12 @@ export default class RollbridgeDaemon {
|
|
|
1791
2029
|
await this.failGenerationTransition(error instanceof Error ? error.message : String(error))
|
|
1792
2030
|
throw error
|
|
1793
2031
|
}
|
|
1794
|
-
if (previousRelease) {
|
|
1795
|
-
const retirementConfig = previousRelease.preserveConfigOnRetirement ? previousRelease.config : release.config
|
|
1796
|
-
|
|
1797
|
-
// Traffic is already on the candidate and singleton ownership has been
|
|
1798
|
-
// transferred. Retirement is deliberately detached so a slow old generation
|
|
1799
|
-
// cannot hold the committed deploy open or race guardian-owned replacement.
|
|
1800
|
-
void this.drainAndPrune(previousRelease, retirementConfig)
|
|
1801
|
-
}
|
|
1802
2032
|
transition.phase = "committed"
|
|
1803
2033
|
transition.error = undefined
|
|
1804
2034
|
transition.journalRevision = (transition.journalRevision ?? 0) + 1
|
|
1805
2035
|
transition.updatedAt = new Date().toISOString()
|
|
1806
2036
|
this.pruneStoppedReleases()
|
|
1807
2037
|
await this.checkpointGenerationTransition()
|
|
1808
|
-
if (previousRelease) {
|
|
1809
|
-
const retirementConfig = !activationLifecycle && previousRelease.preserveConfigOnRetirement ? previousRelease.config : release.config
|
|
1810
|
-
|
|
1811
|
-
// Traffic is already on the healthy candidate. Retirement is deliberately detached
|
|
1812
|
-
// from the owner mutation so a slow quiet/drain hook can never hold the deploy fence
|
|
1813
|
-
// or make the active successor unavailable. drainAndStop applies the bounded force
|
|
1814
|
-
// deadline and records any retirement failure on the old generation.
|
|
1815
|
-
void this.drainAndPrune(previousRelease, retirementConfig)
|
|
1816
|
-
}
|
|
1817
2038
|
}
|
|
1818
2039
|
|
|
1819
2040
|
return {
|
|
@@ -1843,6 +2064,8 @@ export default class RollbridgeDaemon {
|
|
|
1843
2064
|
if (!candidate) throw new Error(`Generation transition candidate ${transition.candidateReleaseId} is not retained`)
|
|
1844
2065
|
if (!previous) throw new Error(`Generation transition previous release ${transition.previousReleaseId} is not retained`)
|
|
1845
2066
|
if (this.activeRelease !== previous) throw new Error(`Previous release ${previous.releaseId} is not the authoritative proxy target`)
|
|
2067
|
+
/** @type {Promise<void> | undefined} */
|
|
2068
|
+
let candidateDrain
|
|
1846
2069
|
|
|
1847
2070
|
if (transition.phase === "activating_candidate") {
|
|
1848
2071
|
if (!transition.error && !transition.activationError) throw new Error("Candidate activation has no recorded failure to compensate")
|
|
@@ -1874,6 +2097,15 @@ export default class RollbridgeDaemon {
|
|
|
1874
2097
|
transition.compensationError = undefined
|
|
1875
2098
|
await this.updateGenerationTransition("restoring_previous")
|
|
1876
2099
|
this.logger("release generation compensation candidate retired", {releaseId: candidate.releaseId})
|
|
2100
|
+
|
|
2101
|
+
// The candidate's bounded drain starts here, before the restoration attempt. The only other
|
|
2102
|
+
// compensation drain runs after the success path, so when incumbent restoration throws the
|
|
2103
|
+
// candidate is left `draining` with a live process: measured 2026-11-17, candidate
|
|
2104
|
+
// `20261117063703` still held `web` pid 3201928 at 07:30Z after its compensation threw at
|
|
2105
|
+
// 06:42:31Z, and reached `release drained` only when a later compensation reached the
|
|
2106
|
+
// success path at 08:47:00Z. A retained generation holding live work is what refuses every
|
|
2107
|
+
// later deploy, and a failed candidate is never a restoration target.
|
|
2108
|
+
candidateDrain = this.drainAndPrune(candidate, candidate.config)
|
|
1877
2109
|
}
|
|
1878
2110
|
|
|
1879
2111
|
if (transition.phase === "restoring_previous") {
|
|
@@ -1885,10 +2117,12 @@ export default class RollbridgeDaemon {
|
|
|
1885
2117
|
|
|
1886
2118
|
transition.compensationError = failure
|
|
1887
2119
|
await this.failGenerationTransition(`Incumbent ${previous.releaseId} restoration failed: ${failure}`)
|
|
2120
|
+
await previous.stopProxyTraffic()
|
|
1888
2121
|
this.logger("release generation compensation restoration failed", {
|
|
1889
2122
|
activationError: transition.activationError,
|
|
1890
2123
|
error: failure,
|
|
1891
|
-
releaseId: previous.releaseId
|
|
2124
|
+
releaseId: previous.releaseId,
|
|
2125
|
+
traffic: "unavailable"
|
|
1892
2126
|
})
|
|
1893
2127
|
throw new Error(`incumbent ${previous.releaseId} restoration failed: ${failure}`, {cause: error})
|
|
1894
2128
|
}
|
|
@@ -1927,7 +2161,7 @@ export default class RollbridgeDaemon {
|
|
|
1927
2161
|
this.logger("release generation compensation checkpoint failed", {error: failure, releaseId: candidate.releaseId})
|
|
1928
2162
|
throw new Error(`compensation checkpoint clear failed: ${failure}`, {cause: error})
|
|
1929
2163
|
}
|
|
1930
|
-
void this.drainAndPrune(candidate, candidate.config)
|
|
2164
|
+
if (candidateDrain === undefined) void this.drainAndPrune(candidate, candidate.config)
|
|
1931
2165
|
this.logger("release generation compensation completed", result)
|
|
1932
2166
|
return result
|
|
1933
2167
|
}
|
|
@@ -1951,6 +2185,19 @@ export default class RollbridgeDaemon {
|
|
|
1951
2185
|
recoveryStatus: "already_recovered"
|
|
1952
2186
|
}
|
|
1953
2187
|
}
|
|
2188
|
+
// A committed journal describes the transition that already published the current
|
|
2189
|
+
// incumbent, not an in-flight transition to recover. When the recovery fence names
|
|
2190
|
+
// that live incumbent, the failed deploy never journaled its own transition, so the
|
|
2191
|
+
// only fence that describes the authoritative release is the completed one: recovery
|
|
2192
|
+
// is already done. In-flight phases keep the stale-fence guard failing closed.
|
|
2193
|
+
if (transition.phase === "committed" && transition.candidateReleaseId === identity.previousReleaseId && this.activeRelease?.releaseId === identity.previousReleaseId && this.releases.get(transition.candidateReleaseId) === this.activeRelease) {
|
|
2194
|
+
return {
|
|
2195
|
+
activeReleaseId: this.activeRelease.releaseId,
|
|
2196
|
+
candidateReleaseId: identity.releaseId,
|
|
2197
|
+
previousReleaseId: identity.previousReleaseId,
|
|
2198
|
+
recoveryStatus: "already_recovered"
|
|
2199
|
+
}
|
|
2200
|
+
}
|
|
1954
2201
|
if (transition.previousReleaseId !== identity.previousReleaseId) {
|
|
1955
2202
|
throw new Error(`Generation transition previous release is ${transition.previousReleaseId}; refusing stale recovery for ${identity.previousReleaseId}`)
|
|
1956
2203
|
}
|
|
@@ -2397,11 +2644,38 @@ export default class RollbridgeDaemon {
|
|
|
2397
2644
|
throw new Error(`Release "${target.releaseId}" is already active.`)
|
|
2398
2645
|
}
|
|
2399
2646
|
|
|
2647
|
+
// A generation recorded as `stopped` whose processes are still alive is the shape a
|
|
2648
|
+
// restore cannot honour: re-deploying that release id re-registers its guardian process
|
|
2649
|
+
// keys under a new provenance while the old processes still hold them, and the guardian
|
|
2650
|
+
// answers with a bare provenance mismatch. Use the guardian's own fence here so the
|
|
2651
|
+
// refusal is decided before anything is touched, and name the divergence — including a
|
|
2652
|
+
// process group that outlived the tracked process and is invisible as a pid — together
|
|
2653
|
+
// with the release the guardian actually reports as active, so the deployment tool can
|
|
2654
|
+
// reconcile its own current pointer instead of restoring a generation that already drained.
|
|
2655
|
+
// A target that is still draining is not refused here: stopping it is what makes its id
|
|
2656
|
+
// reusable, and the deploy below applies the same fence after that stop.
|
|
2657
|
+
const heldGeneration = target.state === "stopped" ? heldGenerationHold(target) : undefined
|
|
2658
|
+
|
|
2659
|
+
if (heldGeneration) {
|
|
2660
|
+
const activeReleaseId = this.activeRelease?.releaseId ?? null
|
|
2661
|
+
|
|
2662
|
+
this.logger("rollback refused: drained generation still holds live processes", {
|
|
2663
|
+
activeReleaseId,
|
|
2664
|
+
heldProcesses: heldGeneration.processes,
|
|
2665
|
+
releaseId: target.releaseId,
|
|
2666
|
+
revision: target.revision
|
|
2667
|
+
})
|
|
2668
|
+
|
|
2669
|
+
throw new Error(`Release "${target.releaseId}" is recorded as drained while ${heldGeneration.count} process${heldGeneration.count === 1 ? "" : "es"} of that generation still hold live work (${heldGeneration.hold.join("; ")}). Restoring it would re-register those processes under a new provenance and fail with a guardian provenance mismatch. The guardian's active release is ${activeReleaseId ? `"${activeReleaseId}"` : "none"}; reconcile the deployment pointer with \`rollbridge status\` before restoring.`)
|
|
2670
|
+
}
|
|
2671
|
+
|
|
2400
2672
|
// The target may still be draining a prior deploy (live processes). Stop it before the
|
|
2401
2673
|
// deploy below re-uses its id in this.releases, otherwise the still-running instance
|
|
2402
2674
|
// would be dropped from status/pruning/shutdown and could be orphaned.
|
|
2675
|
+
// Rollback is an owner mutation too, so this stop carries the same retirement bound as
|
|
2676
|
+
// `stop`: a process the supervisor must not kill must not be able to hold the fence.
|
|
2403
2677
|
if (target.state !== "stopped" && target.state !== "failed") {
|
|
2404
|
-
await target.stop()
|
|
2678
|
+
await target.stop({retirementTimeoutMs: this.config.proxy.nonBlockingDrainTimeoutMs})
|
|
2405
2679
|
}
|
|
2406
2680
|
|
|
2407
2681
|
this.logger("rollback starting", {releaseId: target.releaseId, releasePath: target.releasePath})
|
|
@@ -2622,7 +2896,7 @@ export default class RollbridgeDaemon {
|
|
|
2622
2896
|
|
|
2623
2897
|
/**
|
|
2624
2898
|
* @param {string | undefined} releaseId - Release id, or active release when omitted.
|
|
2625
|
-
* @returns {Promise<void>} Resolves when stopped.
|
|
2899
|
+
* @returns {Promise<void>} Resolves when stopped, or when retirement stops waiting.
|
|
2626
2900
|
*/
|
|
2627
2901
|
async stopRelease(releaseId) {
|
|
2628
2902
|
this.assertNoUnresolvedGenerationTransition("stop")
|
|
@@ -2631,7 +2905,10 @@ export default class RollbridgeDaemon {
|
|
|
2631
2905
|
if (!release) throw new Error(`Release not found: ${releaseId || "active"}`)
|
|
2632
2906
|
if (release === this.activeRelease) this.activeRelease = undefined
|
|
2633
2907
|
|
|
2634
|
-
|
|
2908
|
+
// This command runs inside the owner mutation, so an unbudgeted wait for a process the
|
|
2909
|
+
// supervisor must not kill holds the fence for the whole drain and every later deploy is
|
|
2910
|
+
// refused. The retirement bound is the same one the deploy path already uses.
|
|
2911
|
+
await release.stop({retirementTimeoutMs: this.config.proxy.nonBlockingDrainTimeoutMs})
|
|
2635
2912
|
this.logger("release stopped", {releaseId: release.releaseId})
|
|
2636
2913
|
this.pruneStoppedReleases()
|
|
2637
2914
|
this.persistState()
|
|
@@ -2836,11 +3113,16 @@ export default class RollbridgeDaemon {
|
|
|
2836
3113
|
await this.guardian.retireOwner()
|
|
2837
3114
|
this.guardian.disconnect()
|
|
2838
3115
|
} else {
|
|
3116
|
+
// Without a guardian nobody else can retire these processes, so an unbounded stop would
|
|
3117
|
+
// leave the generation draining forever — the state that made the measured 2026-11-17
|
|
3118
|
+
// rollback refuse. The same retirement budget terminalizes it and names the residue.
|
|
3119
|
+
const retirementTimeoutMs = this.config.proxy.nonBlockingDrainTimeoutMs
|
|
3120
|
+
|
|
2839
3121
|
void Promise.allSettled([
|
|
2840
|
-
...[...this.services.values()].map((processInstance) => processInstance.stop()),
|
|
2841
|
-
...[...this.singletons.values()].map((processInstance) => processInstance.stop()),
|
|
2842
|
-
...[...this.startingReleases].map((release) => release.stop()),
|
|
2843
|
-
...[...this.releases.values()].map((release) => release.stop())
|
|
3122
|
+
...[...this.services.values()].map((processInstance) => processInstance.stop({retirementTimeoutMs})),
|
|
3123
|
+
...[...this.singletons.values()].map((processInstance) => processInstance.stop({retirementTimeoutMs})),
|
|
3124
|
+
...[...this.startingReleases].map((release) => release.stop({retirementTimeoutMs})),
|
|
3125
|
+
...[...this.releases.values()].map((release) => release.stop({retirementTimeoutMs}))
|
|
2844
3126
|
])
|
|
2845
3127
|
}
|
|
2846
3128
|
this.logger("external owner retired", {attestation, status: "draining"})
|
|
@@ -2874,12 +3156,19 @@ export default class RollbridgeDaemon {
|
|
|
2874
3156
|
}
|
|
2875
3157
|
|
|
2876
3158
|
await captureShutdownError(cleanupErrors, "proxy close", async () => this.proxy.close())
|
|
3159
|
+
// Shutdown must return even when a process refuses to exit. The bound is the
|
|
3160
|
+
// `proxy.nonBlockingDrainTimeoutMs` an explicit `rollbridge stop` already uses, and it means
|
|
3161
|
+
// "stop waiting", never "SIGKILL": a process whose graceful window is `"indefinite"` is left
|
|
3162
|
+
// draining rather than killed. Left unbudgeted, that stop keeps the daemon alive after its
|
|
3163
|
+
// control socket is already unlinked — alive, unreachable, and still holding the generation it
|
|
3164
|
+
// stopped: the wedged daemon STOE-2275 measured behind 4,800 `connect ECONNREFUSED` proxy errors.
|
|
3165
|
+
const retirementTimeoutMs = this.config.proxy.nonBlockingDrainTimeoutMs
|
|
2877
3166
|
const dependentStopResults = await Promise.allSettled([
|
|
2878
|
-
...[...this.singletons.values()].map((processInstance) => processInstance.stop()),
|
|
2879
|
-
...[...this.startingReleases].map((release) => release.stop()),
|
|
2880
|
-
...[...this.releases.values()].map((release) => release.stop())
|
|
3167
|
+
...[...this.singletons.values()].map((processInstance) => processInstance.stop({retirementTimeoutMs})),
|
|
3168
|
+
...[...this.startingReleases].map((release) => release.stop({retirementTimeoutMs})),
|
|
3169
|
+
...[...this.releases.values()].map((release) => release.stop({retirementTimeoutMs}))
|
|
2881
3170
|
])
|
|
2882
|
-
const serviceStopResults = await Promise.allSettled([...this.services.values()].map((processInstance) => processInstance.stop()))
|
|
3171
|
+
const serviceStopResults = await Promise.allSettled([...this.services.values()].map((processInstance) => processInstance.stop({retirementTimeoutMs})))
|
|
2883
3172
|
const stopResults = [...dependentStopResults, ...serviceStopResults]
|
|
2884
3173
|
const guardian = this.guardian
|
|
2885
3174
|
if (guardian) await captureShutdownError(cleanupErrors, "process guardian shutdown", () => guardian.shutdown())
|
|
@@ -3063,6 +3352,42 @@ function statusIncludeLogs(value) {
|
|
|
3063
3352
|
return value
|
|
3064
3353
|
}
|
|
3065
3354
|
|
|
3355
|
+
/**
|
|
3356
|
+
* Names the live work a retained release group still holds, using the guardian's own fence.
|
|
3357
|
+
*
|
|
3358
|
+
* A process counts as holding work when the guardian would refuse to hand its key to a new
|
|
3359
|
+
* provenance: a running or stopping process, or a record already `stopped` whose process group
|
|
3360
|
+
* outlived it and is invisible as a pid.
|
|
3361
|
+
* @param {ReleaseGroup} release - Retained generation to inspect.
|
|
3362
|
+
* @returns {HeldGenerationHold | undefined} The hold and its log payload, or undefined when every process is gone.
|
|
3363
|
+
*/
|
|
3364
|
+
function heldGenerationHold(release) {
|
|
3365
|
+
const heldProcesses = release.status().processes.filter((processStatus) => !childRecordIsGone(processStatus))
|
|
3366
|
+
|
|
3367
|
+
if (heldProcesses.length === 0) return undefined
|
|
3368
|
+
|
|
3369
|
+
return {
|
|
3370
|
+
count: heldProcesses.length,
|
|
3371
|
+
hold: heldProcesses.map((processStatus) => {
|
|
3372
|
+
if (processStatus.heldProcessGroup) {
|
|
3373
|
+
const {liveMemberCount, pgid} = processStatus.heldProcessGroup
|
|
3374
|
+
const members = liveMemberCount === undefined ? "" : ` with ${liveMemberCount} live member(s)`
|
|
3375
|
+
|
|
3376
|
+
return `${processStatus.id}: process group ${pgid}${members} still alive after the tracked process exited`
|
|
3377
|
+
}
|
|
3378
|
+
|
|
3379
|
+
return `${processStatus.id}: state ${processStatus.state}, pid ${processStatus.pid ?? "unknown"}`
|
|
3380
|
+
}),
|
|
3381
|
+
processes: heldProcesses.map((processStatus) => ({
|
|
3382
|
+
heldProcessGroupPgid: processStatus.heldProcessGroup?.pgid ?? null,
|
|
3383
|
+
pid: processStatus.pid ?? null,
|
|
3384
|
+
processId: processStatus.id,
|
|
3385
|
+
state: processStatus.state
|
|
3386
|
+
})),
|
|
3387
|
+
state: release.state
|
|
3388
|
+
}
|
|
3389
|
+
}
|
|
3390
|
+
|
|
3066
3391
|
/**
|
|
3067
3392
|
* @param {PrivateOwnerState} transfer - Authenticated private incumbent state.
|
|
3068
3393
|
* @param {GenerationTransition} transition - Unresolved generation transition.
|
|
@@ -3123,6 +3448,15 @@ export function isLegacyGuardianPrepareDiagnostic(diagnostic) {
|
|
|
3123
3448
|
diagnostic === "Unknown guardian command: prepare-owner-replacement"
|
|
3124
3449
|
}
|
|
3125
3450
|
|
|
3451
|
+
/**
|
|
3452
|
+
* Reports the capability diagnostics that identify a guardian predating daemon recovery.
|
|
3453
|
+
* @param {string} diagnostic - Guardian error message.
|
|
3454
|
+
* @returns {boolean} Whether the guardian predates the daemon recovery capability.
|
|
3455
|
+
*/
|
|
3456
|
+
function isGuardianDaemonRecoveryDiagnostic(diagnostic) {
|
|
3457
|
+
return diagnostic === "Guardian capabilities requires a process key" || diagnostic === "Unknown guardian command: capabilities"
|
|
3458
|
+
}
|
|
3459
|
+
|
|
3126
3460
|
/**
|
|
3127
3461
|
* @param {DaemonStatus} snapshot - Serialized owner process snapshot.
|
|
3128
3462
|
* @param {Record<string, string>} [singletonReleaseIds] - Exact singleton owner releases.
|
|
@@ -3445,7 +3779,7 @@ function requiredNonNegativeInteger(value, key) {
|
|
|
3445
3779
|
/**
|
|
3446
3780
|
* Selects stopped releases to prune by the retention policy, keeping the most recent.
|
|
3447
3781
|
* @param {PrunableRelease[]} releases - Status of all tracked releases, in deploy order (oldest first).
|
|
3448
|
-
* @param {
|
|
3782
|
+
* @param {{keep: number, maxAgeMs: number}} policy - Retention policy; draining generations are never pruned here.
|
|
3449
3783
|
* @param {number} now - Current epoch milliseconds.
|
|
3450
3784
|
* @returns {string[]} Release ids to remove.
|
|
3451
3785
|
*/
|
|
@@ -3470,7 +3804,7 @@ export function releasesToPrune(releases, policy, now) {
|
|
|
3470
3804
|
}
|
|
3471
3805
|
|
|
3472
3806
|
/**
|
|
3473
|
-
* @typedef {{alive: boolean, application?: string, activeReleaseId?: string | null, proxy?: {host: string, port: number}}} ControlSocketInspection
|
|
3807
|
+
* @typedef {{alive: boolean, application?: string, activeReleaseId?: string | null, guardian?: {ownerMutation: Record<string, JsonValue> | null, version: string | null}, proxy?: {host: string, port: number}}} ControlSocketInspection
|
|
3474
3808
|
*/
|
|
3475
3809
|
|
|
3476
3810
|
/**
|
|
@@ -3545,7 +3879,7 @@ export async function inspectControlSocket(socketPath, timeoutMs = 1000) {
|
|
|
3545
3879
|
/**
|
|
3546
3880
|
* Parses a control status response line into a Rollbridge identity, if it is one.
|
|
3547
3881
|
* @param {string} line - JSON response line.
|
|
3548
|
-
* @returns {{application: string, activeReleaseId: string | null, proxy: {host: string, port: number} | undefined} | undefined} Identity, or undefined when unrecognized.
|
|
3882
|
+
* @returns {{application: string, activeReleaseId: string | null, guardian: {ownerMutation: Record<string, JsonValue> | null, version: string | null} | undefined, proxy: {host: string, port: number} | undefined} | undefined} Identity, or undefined when unrecognized.
|
|
3549
3883
|
*/
|
|
3550
3884
|
function parseControlStatus(line) {
|
|
3551
3885
|
/** @type {JsonValue} */
|
|
@@ -3562,5 +3896,8 @@ function parseControlStatus(line) {
|
|
|
3562
3896
|
|
|
3563
3897
|
const proxy = "proxy" in parsed && parsed.proxy && typeof parsed.proxy === "object" && !Array.isArray(parsed.proxy) && typeof parsed.proxy.host === "string" && typeof parsed.proxy.port === "number" ? {host: parsed.proxy.host, port: parsed.proxy.port} : undefined
|
|
3564
3898
|
|
|
3565
|
-
|
|
3899
|
+
const guardianValue = "guardian" in parsed && parsed.guardian && typeof parsed.guardian === "object" && !Array.isArray(parsed.guardian) ? parsed.guardian : undefined
|
|
3900
|
+
const guardian = guardianValue && typeof guardianValue.version === "string" || guardianValue && guardianValue.version === null ? {ownerMutation: guardianValue.ownerMutation && typeof guardianValue.ownerMutation === "object" && !Array.isArray(guardianValue.ownerMutation) ? guardianValue.ownerMutation : null, version: guardianValue.version} : undefined
|
|
3901
|
+
|
|
3902
|
+
return {activeReleaseId: typeof parsed.activeReleaseId === "string" ? parsed.activeReleaseId : null, application: parsed.application, guardian, proxy}
|
|
3566
3903
|
}
|