rollbridge 0.1.73 → 0.1.74
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +63 -10
- package/TODO.md +13 -0
- package/changelog.d/20261025-guardian-re-register-dead-generation.md +13 -0
- package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +1 -0
- package/changelog.d/20261104-compose-service-units.md +21 -0
- package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +20 -0
- package/changelog.d/20261106-release-dependency-boot-budgets.md +32 -0
- package/changelog.d/20261109-path-filtered-builds.md +11 -0
- package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +26 -0
- package/changelog.d/20261111-contention-sensitive-daemon-specs.md +27 -0
- package/changelog.d/20261113-bounded-nonblocking-retirement.md +5 -0
- package/changelog.d/20261115-bound-draining-release-groups.md +62 -0
- package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +27 -0
- package/changelog.d/20261115-wedged-generation-escalation.md +44 -0
- package/changelog.d/20261116-bounded-explicit-stop-retirement.md +30 -0
- package/changelog.d/20261116-config-json-canonical-shape.md +11 -0
- package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +29 -0
- package/changelog.d/20261116-guardian-version-handoff.md +37 -0
- package/changelog.d/20261116-reclaim-dead-owner-mutation.md +7 -0
- package/changelog.d/20261117-activate-before-incumbent-retirement.md +1 -0
- package/changelog.d/20261117-activation-path-drain-deadline.md +35 -0
- package/changelog.d/20261117-stop-companion-after-retired-dependency.md +7 -0
- package/changelog.d/20261118-incumbent-retirement-authority-guard.md +1 -0
- package/changelog.d/20261118-owner-retirement-no-guardian-bound.md +19 -0
- package/changelog.d/20261118-shutdown-retirement-bound-restored.md +31 -0
- package/docs/cli.md +128 -3
- package/docs/compose-services.md +222 -0
- package/docs/config.md +177 -6
- package/docs/troubleshooting.md +97 -6
- package/docs/velocious.md +1 -1
- package/docs/workers.md +11 -0
- package/examples/tensorbuzz.com.js +3 -2
- package/package.json +1 -1
- package/src/cli.js +68 -2
- package/src/compose-unit.js +725 -0
- package/src/config.js +346 -8
- package/src/daemon.js +411 -78
- package/src/doctor.js +74 -0
- package/src/drain-policy.js +252 -0
- package/src/guardian-client.js +68 -23
- package/src/health.js +99 -8
- package/src/managed-process.js +198 -45
- package/src/process-guardian.js +212 -37
- package/src/process-memory.js +58 -23
- package/src/process-order.js +26 -0
- package/src/release-group.js +438 -45
- package/tensorbuzz.yml +20 -1
- package/test/completion.test.js +5 -2
- package/test/compose-config.test.js +160 -0
- package/test/compose-docker.test.js +297 -0
- package/test/compose-unit.test.js +415 -0
- package/test/config-validation.test.js +135 -5
- package/test/drain-policy.test.js +328 -0
- package/test/fixtures/deferred-boot-app.js +106 -0
- package/test/fixtures/gated-app.js +50 -0
- package/test/guardian-client.test.js +119 -5
- package/test/health.test.js +74 -1
- package/test/managed-process.test.js +260 -24
- package/test/owner-recovery.test.js +125 -21
- package/test/owner-replacement.test.js +175 -14
- package/test/process-memory.test.js +47 -1
- package/test/release-group.test.js +408 -2
- package/test/rollbridge.test.js +774 -72
- package/test/shutdown-completion.test.js +119 -0
- package/test/shutdown-retirement-bound.test.js +162 -0
- package/test/support/process.js +3 -1
- package/test/tensorbuzz-config.test.js +228 -0
package/src/daemon.js
CHANGED
|
@@ -9,8 +9,10 @@ import {isDeepStrictEqual} from "node:util"
|
|
|
9
9
|
import httpProxy from "http-proxy"
|
|
10
10
|
import {loadConfig} from "./config.js"
|
|
11
11
|
import {openControlSession} from "./control-client.js"
|
|
12
|
+
import {admitGenerationRelease} from "./drain-policy.js"
|
|
12
13
|
import EventLog from "./event-log.js"
|
|
13
|
-
import GuardianClient from "./guardian-client.js"
|
|
14
|
+
import GuardianClient, {guardianVersionFromProcess, usableSocketPath} from "./guardian-client.js"
|
|
15
|
+
import {childRecordIsGone} from "./managed-process.js"
|
|
14
16
|
import ReleaseGroup from "./release-group.js"
|
|
15
17
|
import {clearState, isProcessAlive, liveProcesses, readState, writeState} from "./state-store.js"
|
|
16
18
|
import {resolveGroupId, resolveUserId} from "./system-ids.js"
|
|
@@ -23,18 +25,20 @@ const STATE_PERSIST_INTERVAL_MS = 5000
|
|
|
23
25
|
* @typedef {{releaseId?: string, releasePath: string, revision?: string}} DeployArgs
|
|
24
26
|
* @typedef {{attestation?: string, releaseId: string, releasePath: string, revision: string}} BootstrapIdentity
|
|
25
27
|
* @typedef {{id: string, process: import("./managed-process.js").ManagedProcessStatus}} ProcessStatus
|
|
26
|
-
* @typedef {{disruptive:
|
|
28
|
+
* @typedef {{disruptive: true, mode: "guardian-version-handoff" | "legacy-first-upgrade", reason: string}} OwnerTransition
|
|
29
|
+
* @typedef {"guardian-version-drift" | "legacy-protocol"} GuardianHandoffProtocol
|
|
27
30
|
* @typedef {"candidate_ready" | "retiring_previous" | "previous_retired" | "activating_candidate" | "restoring_previous" | "retiring_failed_candidate" | "degraded_active" | "committed_pending" | "committed" | "restoring_committed"} GenerationTransitionPhase
|
|
28
31
|
* @typedef {{activationError?: string, activationLifecycle?: boolean, candidateReleaseId: string, candidateReleasePath: string, candidateRevision: string, compensationError?: string, configDigest: string, degradedIncumbent?: boolean, error?: string, journalRevision?: number, phase: GenerationTransitionPhase, previousReleaseId: string | null, startedAt: string, updatedAt: string}} GenerationTransition
|
|
29
|
-
* @typedef {{activeReleaseId: string | null, application: string, bootstrap: BootstrapIdentity | undefined, control: import("./config.js").ControlConfig, daemonPid: number, daemonRuntime: import("./daemon-runtime.js").DaemonRuntimeIdentity | undefined, generationTransition?: GenerationTransition, ownerRecovery: {configDigest: string, ready: boolean} | undefined, ownerTransition?: OwnerTransition, orphans: {id: string, pid: number, releaseId: string | null}[], proxy: {host: string, port: number | undefined, upstreamHost: string}, releaseReferences: {releaseId: string, releasePath: string}[], releases: import("./release-group.js").ReleaseStatus[], services: ProcessStatus[], singletons: ProcessStatus[]}} DaemonStatus
|
|
32
|
+
* @typedef {{activeReleaseId: string | null, application: string, bootstrap: BootstrapIdentity | undefined, control: import("./config.js").ControlConfig, daemonPid: number, daemonRuntime: import("./daemon-runtime.js").DaemonRuntimeIdentity | undefined, generationTransition?: GenerationTransition, guardian?: {guardianPid: number | null, ownerMutation: Record<string, JsonValue> | null, version: string | null}, ownerRecovery: {configDigest: string, ready: boolean} | undefined, ownerTransition?: OwnerTransition, orphans: {id: string, pid: number, releaseId: string | null}[], proxy: {host: string, port: number | undefined, upstreamHost: string}, releaseReferences: {releaseId: string, releasePath: string}[], releases: import("./release-group.js").ReleaseStatus[], services: ProcessStatus[], singletons: ProcessStatus[]}} DaemonStatus
|
|
30
33
|
* @typedef {Omit<import("./managed-process.js").ManagedProcessStatus, "logs">} ManagedProcessStatusWithoutLogs
|
|
31
34
|
* @typedef {Omit<import("./release-group.js").ReleaseStatus, "processes"> & {processes: ManagedProcessStatusWithoutLogs[]}} ReleaseStatusWithoutLogs
|
|
32
35
|
* @typedef {Omit<ProcessStatus, "process"> & {process: ManagedProcessStatusWithoutLogs}} ProcessStatusWithoutLogs
|
|
33
36
|
* @typedef {Omit<DaemonStatus, "releases" | "services" | "singletons"> & {releases: ReleaseStatusWithoutLogs[], services: ProcessStatusWithoutLogs[], singletons: ProcessStatusWithoutLogs[]}} DaemonStatusWithoutLogs
|
|
37
|
+
* @typedef {{count: number, hold: string[], processes: {heldProcessGroupPgid: number | null, pid: number | null, processId: string, state: import("./managed-process.js").ManagedProcessState}[], state: import("./release-group.js").ReleaseState}} HeldGenerationHold
|
|
34
38
|
* @typedef {{configDigest: string, format: number, guardian: {pid?: number, socketPath: string, token: string}, reconnectGraceMs: number}} OwnerRecoveryMetadata
|
|
35
39
|
* @typedef {DaemonStatus & {recovery: OwnerRecoveryMetadata, serviceReleaseIds?: Record<string, string>, singletonReleaseIds?: Record<string, string>}} OwnerRecoverySnapshot
|
|
36
40
|
* @typedef {{authority: JsonValue, config: import("./config.js").RollbridgeConfig, listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>, listenerSourceId?: string, recovery?: {command: JsonValue, reconnectGraceMs: number, startupTimeoutMs: number}, releaseConfigs?: Record<string, import("./config.js").RollbridgeConfig>, retiredListenerHandoff?: number, serviceReleaseIds?: Record<string, string>, singletonReleaseIds?: Record<string, string>, snapshot: OwnerRecoverySnapshot}} PrivateOwnerState
|
|
37
|
-
* @typedef {{boundaryCrossed: boolean, incumbentControl?: Awaited<ReturnType<typeof openControlSession>>, incumbentStartTime: string, legacyGuardian: GuardianClient, legacyInventory: {key: string, provenance: string}[], legacyPrepared?: {ownerState: JsonValue, replacementId: string}, legacySnapshot: OwnerRecoverySnapshot, prepared: {ownerState: JsonValue, replacementId: string}, recoverySnapshot: OwnerRecoverySnapshot}} LegacyOwnerBridge
|
|
41
|
+
* @typedef {{boundaryCrossed: boolean, guardianVersion?: string, incumbentControl?: Awaited<ReturnType<typeof openControlSession>>, incumbentStartTime: string, legacyGuardian: GuardianClient, legacyInventory: {key: string, provenance: string}[], legacyPrepared?: {ownerState: JsonValue, replacementId: string}, legacySnapshot: OwnerRecoverySnapshot, prepared: {ownerState: JsonValue, replacementId: string}, protocol: GuardianHandoffProtocol, recoverySnapshot: OwnerRecoverySnapshot}} LegacyOwnerBridge
|
|
38
42
|
*/
|
|
39
43
|
|
|
40
44
|
export default class RollbridgeDaemon {
|
|
@@ -146,7 +150,7 @@ export default class RollbridgeDaemon {
|
|
|
146
150
|
this.logger("release generation transition recovery failed", {error: error instanceof Error ? error.message : String(error), releaseId: transition.candidateReleaseId})
|
|
147
151
|
}
|
|
148
152
|
return {}
|
|
149
|
-
})
|
|
153
|
+
}, {waitingFor: "the retained release generations to settle"})
|
|
150
154
|
} finally {
|
|
151
155
|
this.generationTransitionRecovery = false
|
|
152
156
|
}
|
|
@@ -203,7 +207,7 @@ export default class RollbridgeDaemon {
|
|
|
203
207
|
if (!snapshot || snapshot.recovery?.guardian || !exactBootstrap) throw new Error("Retired owner reset requires an exact committed bootstrap with missing guardian identity")
|
|
204
208
|
const control = await inspectControlSocket(this.config.control.path)
|
|
205
209
|
if (control.alive) throw new Error(`Retired owner reset refuses responsive control owner at ${this.config.control.path}`)
|
|
206
|
-
this.guardianIdentity = {socketPath: `${this.statePath}.guardian.sock
|
|
210
|
+
this.guardianIdentity = {socketPath: usableSocketPath(`${this.statePath}.guardian.sock`), token: crypto.randomBytes(32).toString("hex")}
|
|
207
211
|
this.guardian = new GuardianClient(this.guardianIdentity)
|
|
208
212
|
await this.guardian.launch()
|
|
209
213
|
this.guardianIdentity.pid = this.guardian.pid
|
|
@@ -221,10 +225,12 @@ export default class RollbridgeDaemon {
|
|
|
221
225
|
|
|
222
226
|
if (snapshot && !recovery) throw new Error(`Owner recovery state ${this.statePath} is missing durable guardian identity; refusing to overwrite it.`)
|
|
223
227
|
|
|
224
|
-
const guardianIdentity = recovery?.guardian
|
|
225
|
-
socketPath:
|
|
226
|
-
|
|
227
|
-
|
|
228
|
+
const guardianIdentity = recovery?.guardian
|
|
229
|
+
? {...recovery.guardian, socketPath: usableSocketPath(recovery.guardian.socketPath)}
|
|
230
|
+
: {
|
|
231
|
+
socketPath: usableSocketPath(`${this.statePath}.guardian.sock`),
|
|
232
|
+
token: crypto.randomBytes(32).toString("hex")
|
|
233
|
+
}
|
|
228
234
|
this.guardianIdentity = guardianIdentity
|
|
229
235
|
this.guardian = new GuardianClient(guardianIdentity)
|
|
230
236
|
if (recovery) {
|
|
@@ -406,11 +412,7 @@ export default class RollbridgeDaemon {
|
|
|
406
412
|
shouldStart: () => !this.stopping
|
|
407
413
|
})
|
|
408
414
|
|
|
409
|
-
await release.restore(releaseStatus, {
|
|
410
|
-
adoptCommittedGeneration: releaseStatus.releaseId === snapshot.activeReleaseId && snapshot.generationTransition?.phase === "committed" && snapshot.ownerTransition?.mode === "guardian-version-handoff",
|
|
411
|
-
configDigest: ownerConfigDigest(config),
|
|
412
|
-
synchronizeLifecycleRole: synchronizeLifecycleRoles
|
|
413
|
-
})
|
|
415
|
+
await release.restore(releaseStatus, {synchronizeLifecycleRole: synchronizeLifecycleRoles})
|
|
414
416
|
this.releases.set(release.releaseId, release)
|
|
415
417
|
if (release.releaseId === snapshot.activeReleaseId) this.activeRelease = release
|
|
416
418
|
}
|
|
@@ -504,15 +506,36 @@ export default class RollbridgeDaemon {
|
|
|
504
506
|
/**
|
|
505
507
|
* Prepares from private guardian state, starts listeners, then asks the committed owner
|
|
506
508
|
* to atomically fence itself and transfer authority.
|
|
509
|
+
* @returns {Promise<void>} Resolves after the replacement is committed.
|
|
507
510
|
*/
|
|
508
511
|
async replaceIncompatibleOwner() {
|
|
512
|
+
const maxAttempts = 3
|
|
513
|
+
let lastError
|
|
514
|
+
|
|
515
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
|
516
|
+
try {
|
|
517
|
+
return await this.replaceIncompatibleOwnerAttempt()
|
|
518
|
+
} catch (error) {
|
|
519
|
+
lastError = error
|
|
520
|
+
const diagnostic = error instanceof Error ? error.message : String(error)
|
|
521
|
+
|
|
522
|
+
if (!/^Guardian owner state changed after prepare \(prepared revision \d+, current revision \d+\); abort the stale candidate before preparing again$/.test(diagnostic) || attempt === maxAttempts) {
|
|
523
|
+
throw error
|
|
524
|
+
}
|
|
525
|
+
this.logger("owner replacement candidate became stale; re-preparing", {attempt, maxAttempts, error: diagnostic})
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
throw lastError
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
async replaceIncompatibleOwnerAttempt() {
|
|
509
532
|
if (!this.statePath || !this.config.ownerRecovery) throw new Error("Atomic owner replacement requires ownerRecovery and the same statePath transaction anchor")
|
|
510
533
|
this.controlCommandsReady = false
|
|
511
534
|
const state = await readState(this.statePath)
|
|
512
535
|
const persisted = state && typeof state === "object" && !Array.isArray(state) ? /** @type {OwnerRecoverySnapshot} */ (state) : undefined
|
|
513
536
|
|
|
514
537
|
if (!persisted?.recovery?.guardian) throw new Error(`No committed durable owner transaction was found at ${this.statePath}`)
|
|
515
|
-
this.guardianIdentity = persisted.recovery.guardian
|
|
538
|
+
this.guardianIdentity = {...persisted.recovery.guardian, socketPath: usableSocketPath(persisted.recovery.guardian.socketPath)}
|
|
516
539
|
this.guardian = new GuardianClient(this.guardianIdentity)
|
|
517
540
|
await this.guardian.connect()
|
|
518
541
|
this.watchOwnerReplacementEvents()
|
|
@@ -520,9 +543,16 @@ export default class RollbridgeDaemon {
|
|
|
520
543
|
configDigest: persisted.recovery.configDigest,
|
|
521
544
|
runtime: persisted.daemonRuntime ? {...persisted.daemonRuntime} : null
|
|
522
545
|
}
|
|
523
|
-
const
|
|
546
|
+
const guardianVersion = await this.retainedGuardianVersion()
|
|
547
|
+
const versionDrift = Boolean(this.runtime?.version && guardianVersion && guardianVersion !== this.runtime.version)
|
|
548
|
+
const replacementProtocol = versionDrift ? "legacy" : await this.guardian.ownerReplacementProtocol()
|
|
524
549
|
const legacyBridge = replacementProtocol === "legacy"
|
|
525
|
-
? await this.prepareLegacyOwnerReplacement({
|
|
550
|
+
? await this.prepareLegacyOwnerReplacement({
|
|
551
|
+
guardianVersion,
|
|
552
|
+
persisted,
|
|
553
|
+
persistedAuthority,
|
|
554
|
+
protocol: versionDrift ? "guardian-version-drift" : "legacy-protocol"
|
|
555
|
+
})
|
|
526
556
|
: undefined
|
|
527
557
|
const prepared = legacyBridge?.prepared ?? await this.guardian.prepareOwnerReplacement(persistedAuthority, this.ownerAuthority())
|
|
528
558
|
let preparedStatus
|
|
@@ -768,6 +798,33 @@ export default class RollbridgeDaemon {
|
|
|
768
798
|
}
|
|
769
799
|
}
|
|
770
800
|
|
|
801
|
+
/**
|
|
802
|
+
* Reads the retained guardian's package version without deciding the replacement protocol.
|
|
803
|
+
*
|
|
804
|
+
* The version is read from the guardian's own executable rather than its capabilities
|
|
805
|
+
* command: a pre-replacement guardian only answers that command once a replacement
|
|
806
|
+
* transaction is prepared, so asking here would move the probe ahead of the transaction
|
|
807
|
+
* every capability-gated guardian depends on. A guardian whose version cannot be read
|
|
808
|
+
* stays on the protocol classification path.
|
|
809
|
+
* @returns {Promise<string | undefined>} Retained guardian package version.
|
|
810
|
+
*/
|
|
811
|
+
async retainedGuardianVersion() {
|
|
812
|
+
if (!this.guardian) throw new Error("Guardian version reconciliation requires the durable process guardian")
|
|
813
|
+
return await guardianVersionFromProcess(this.guardian.pid)
|
|
814
|
+
}
|
|
815
|
+
|
|
816
|
+
/**
|
|
817
|
+
* Names the last-resort refusal for a guardian this daemon cannot hand off automatically.
|
|
818
|
+
* @param {string | undefined} guardianVersion - Retained guardian package version.
|
|
819
|
+
* @param {string} cause - Measured reason the automatic handoff cannot run.
|
|
820
|
+
* @returns {Error} Refusal naming both versions and the exact repair command.
|
|
821
|
+
*/
|
|
822
|
+
guardianVersionHandoffRefusal(guardianVersion, cause) {
|
|
823
|
+
const configArgument = this.configPath ? ` --config ${this.configPath}` : ""
|
|
824
|
+
|
|
825
|
+
return new Error(`The persistent Rollbridge guardian is version ${guardianVersion ?? "unknown"} while this daemon runs rollbridge ${this.runtime?.version ?? "unknown"}; ${cause}. Run "rollbridge shutdown${configArgument}" and then "rollbridge ensure-daemon${configArgument}" to replace the supervisor with this Rollbridge installation.`)
|
|
826
|
+
}
|
|
827
|
+
|
|
771
828
|
/** Requires the persistent guardian protocol which can restart an accepted daemon. */
|
|
772
829
|
async assertGuardianDaemonRecoveryCapability() {
|
|
773
830
|
if (!this.guardian) throw new Error("Owner recovery capability check requires the durable process guardian")
|
|
@@ -776,10 +833,18 @@ export default class RollbridgeDaemon {
|
|
|
776
833
|
try {
|
|
777
834
|
capabilities = await this.guardian.capabilities()
|
|
778
835
|
} catch (error) {
|
|
779
|
-
if (!(error instanceof Error) || !
|
|
836
|
+
if (!(error instanceof Error) || !isGuardianDaemonRecoveryDiagnostic(error.message)) throw error
|
|
780
837
|
throw new Error("The persistent Rollbridge guardian predates daemon recovery. Perform one explicit clean Rollbridge shutdown and restart before retrying this package upgrade.", {cause: error})
|
|
781
838
|
}
|
|
782
839
|
if (capabilities.daemonRecovery !== 1) throw new Error("The persistent Rollbridge guardian does not support the required daemon recovery protocol")
|
|
840
|
+
const clientVersion = this.runtime?.version
|
|
841
|
+
const guardianVersion = capabilities.guardianVersion
|
|
842
|
+
|
|
843
|
+
if (!clientVersion) return
|
|
844
|
+
if (!guardianVersion) throw this.guardianVersionHandoffRefusal(guardianVersion, "its version cannot be attested, so no handoff can be authorised")
|
|
845
|
+
if (guardianVersion !== clientVersion) {
|
|
846
|
+
throw this.guardianVersionHandoffRefusal(guardianVersion, "this startup path has no incumbent daemon to hand guardian ownership from")
|
|
847
|
+
}
|
|
783
848
|
}
|
|
784
849
|
|
|
785
850
|
/**
|
|
@@ -900,13 +965,15 @@ export default class RollbridgeDaemon {
|
|
|
900
965
|
}
|
|
901
966
|
|
|
902
967
|
/**
|
|
903
|
-
* Authenticates and prepares the
|
|
904
|
-
*
|
|
968
|
+
* Authenticates and prepares the disruptive guardian-upgrade bridge, for a guardian that predates
|
|
969
|
+
* the replacement protocol or whose package version drifted from this daemon's runtime.
|
|
970
|
+
* @param {{guardianVersion?: string, persisted: OwnerRecoverySnapshot, persistedAuthority: {configDigest: string, runtime: import("./daemon-runtime.js").DaemonRuntimeIdentity | null}, protocol: GuardianHandoffProtocol}} options - Bridge evidence and why the bridge is required.
|
|
905
971
|
* @returns {Promise<LegacyOwnerBridge>} Prepared bridge.
|
|
906
972
|
*/
|
|
907
|
-
async prepareLegacyOwnerReplacement({persisted, persistedAuthority}) {
|
|
973
|
+
async prepareLegacyOwnerReplacement({guardianVersion, persisted, persistedAuthority, protocol}) {
|
|
908
974
|
const legacyProcessKey = ownerSnapshotProcessKeys(persisted, persisted.singletonReleaseIds)[0]
|
|
909
975
|
const legacyGuardian = this.guardian
|
|
976
|
+
const versionHandoff = protocol === "guardian-version-drift"
|
|
910
977
|
let legacyPrepared
|
|
911
978
|
|
|
912
979
|
if (!legacyProcessKey) throw new Error("Legacy disruptive owner replacement requires an exact guardian-owned process registration in durable state")
|
|
@@ -939,12 +1006,23 @@ export default class RollbridgeDaemon {
|
|
|
939
1006
|
|
|
940
1007
|
try {
|
|
941
1008
|
if (persisted.recovery.configDigest !== this.ownerRecoveryConfigDigest()) {
|
|
1009
|
+
if (versionHandoff) throw this.guardianVersionHandoffRefusal(guardianVersion, "the handoff keeps the incumbent config identity unchanged and this deploy changes config authority")
|
|
942
1010
|
throw new Error("The one-time legacy guardian bridge requires the incumbent config identity unchanged; retry the config change after the bridge establishes split-3 authority")
|
|
943
1011
|
}
|
|
944
|
-
if (!this.legacyIncumbentPid)
|
|
1012
|
+
if (!this.legacyIncumbentPid) {
|
|
1013
|
+
if (versionHandoff) throw this.guardianVersionHandoffRefusal(guardianVersion, "ensure-daemon reported no incumbent daemon to hand guardian ownership from")
|
|
1014
|
+
throw new Error("The authenticated legacy guardian requires an exact incumbent PID from ensure-daemon for the disruptive bridge")
|
|
1015
|
+
}
|
|
945
1016
|
if (!this.guardianIdentity?.pid) throw new Error("The authenticated legacy guardian state is missing its exact guardian PID")
|
|
946
1017
|
await verifyLegacyGuardianProcess(this.guardianIdentity.pid, this.guardianIdentity.socketPath)
|
|
947
|
-
|
|
1018
|
+
let incumbentStartTime
|
|
1019
|
+
|
|
1020
|
+
try {
|
|
1021
|
+
incumbentStartTime = await verifyLegacyDaemonProcess(this.legacyIncumbentPid, this.configPath, persisted.control.path)
|
|
1022
|
+
} catch (error) {
|
|
1023
|
+
if (!versionHandoff) throw error
|
|
1024
|
+
throw this.guardianVersionHandoffRefusal(guardianVersion, `the incumbent daemon PID ${this.legacyIncumbentPid} is not the attested owner: ${error instanceof Error ? error.message : String(error)}`)
|
|
1025
|
+
}
|
|
948
1026
|
if (legacyPrepared) assertLegacyPrivateOwnerState(legacyPrepared.ownerState, persisted, persistedAuthority)
|
|
949
1027
|
try {
|
|
950
1028
|
incumbentControl = await openControlSession(persisted.control.path)
|
|
@@ -971,7 +1049,9 @@ export default class RollbridgeDaemon {
|
|
|
971
1049
|
|
|
972
1050
|
assertLegacyGuardianInventoryMembership(legacyInventory, /** @type {PrivateOwnerState} */ (ownerState))
|
|
973
1051
|
const upgradedIdentity = /** @type {{pid?: number, socketPath: string, token: string}} */ ({
|
|
974
|
-
socketPath:
|
|
1052
|
+
socketPath: usableSocketPath(versionHandoff
|
|
1053
|
+
? `${this.statePath}.guardian-${this.runtime?.version}.sock`
|
|
1054
|
+
: `${this.statePath}.split3-guardian.sock`),
|
|
975
1055
|
token: crypto.randomBytes(32).toString("hex")
|
|
976
1056
|
})
|
|
977
1057
|
|
|
@@ -986,12 +1066,15 @@ export default class RollbridgeDaemon {
|
|
|
986
1066
|
recovery: {...persisted.recovery, guardian: upgradedIdentity}
|
|
987
1067
|
})
|
|
988
1068
|
|
|
989
|
-
this.logger("
|
|
1069
|
+
this.logger("guardian upgrade bridge prepared", {
|
|
990
1070
|
guardianPid: upgradedIdentity.pid ?? null,
|
|
1071
|
+
guardianVersion: guardianVersion ?? "unknown",
|
|
991
1072
|
incumbentPid: this.legacyIncumbentPid,
|
|
992
|
-
|
|
1073
|
+
protocol,
|
|
1074
|
+
replacementId: prepared.replacementId,
|
|
1075
|
+
runtimeVersion: this.runtime?.version ?? "unknown"
|
|
993
1076
|
})
|
|
994
|
-
return {boundaryCrossed: false, incumbentControl, incumbentStartTime, legacyGuardian, legacyInventory, legacyPrepared, legacySnapshot: persisted, prepared, recoverySnapshot}
|
|
1077
|
+
return {boundaryCrossed: false, guardianVersion, incumbentControl, incumbentStartTime, legacyGuardian, legacyInventory, legacyPrepared, legacySnapshot: persisted, prepared, protocol, recoverySnapshot}
|
|
995
1078
|
} catch (upgradeError) {
|
|
996
1079
|
incumbentControl?.close()
|
|
997
1080
|
let cleanupError
|
|
@@ -1071,10 +1154,20 @@ export default class RollbridgeDaemon {
|
|
|
1071
1154
|
|
|
1072
1155
|
if (staged.committed) throw new Error("Legacy guardian committed replacement before the authenticated disruptive boundary")
|
|
1073
1156
|
}
|
|
1074
|
-
|
|
1157
|
+
const boundaryReason = bridge.protocol === "guardian-version-drift"
|
|
1158
|
+
? `retained guardian ${bridge.guardianVersion ?? "unknown"} predates this daemon runtime ${this.runtime?.version ?? "unknown"}`
|
|
1159
|
+
: "retained guardian and daemon lacked atomic replacement protocol"
|
|
1160
|
+
|
|
1161
|
+
this.logger("guardian upgrade bridge crossed", {
|
|
1075
1162
|
disruptive: true,
|
|
1163
|
+
guardianVersion: bridge.guardianVersion ?? "unknown",
|
|
1076
1164
|
incumbentPid: this.legacyIncumbentPid,
|
|
1077
|
-
|
|
1165
|
+
protocol: bridge.protocol,
|
|
1166
|
+
reason: boundaryReason,
|
|
1167
|
+
// The retained guardian keeps its own children; the boundary retires only the incumbent daemon.
|
|
1168
|
+
retainedGuardianPid: bridge.legacyGuardian.pid ?? null,
|
|
1169
|
+
retainedGuardianProcesses: bridge.legacyInventory.length,
|
|
1170
|
+
runtimeVersion: this.runtime?.version ?? "unknown"
|
|
1078
1171
|
})
|
|
1079
1172
|
await this.guardian.beginLegacyOwnerClaim(
|
|
1080
1173
|
bridge.prepared.replacementId,
|
|
@@ -1090,8 +1183,8 @@ export default class RollbridgeDaemon {
|
|
|
1090
1183
|
await this.guardian.completeLegacyOwnerClaim(bridge.prepared.replacementId)
|
|
1091
1184
|
this.ownerTransition = /** @type {OwnerTransition} */ ({
|
|
1092
1185
|
disruptive: true,
|
|
1093
|
-
mode: "legacy-first-upgrade",
|
|
1094
|
-
reason:
|
|
1186
|
+
mode: bridge.protocol === "guardian-version-drift" ? "guardian-version-handoff" : "legacy-first-upgrade",
|
|
1187
|
+
reason: boundaryReason
|
|
1095
1188
|
})
|
|
1096
1189
|
}
|
|
1097
1190
|
|
|
@@ -1393,7 +1486,7 @@ export default class RollbridgeDaemon {
|
|
|
1393
1486
|
releaseId: stringOrUndefined(data.releaseId),
|
|
1394
1487
|
releasePath: requiredString(data.releasePath, "releasePath"),
|
|
1395
1488
|
revision: stringOrUndefined(data.revision)
|
|
1396
|
-
}))
|
|
1489
|
+
}), {waitingFor: "candidate activation and the retired generation handoff"})
|
|
1397
1490
|
}
|
|
1398
1491
|
|
|
1399
1492
|
if (commandName === "recover-generation-transition") {
|
|
@@ -1403,11 +1496,22 @@ export default class RollbridgeDaemon {
|
|
|
1403
1496
|
releaseId: requiredString(data.releaseId, "releaseId"),
|
|
1404
1497
|
releasePath: requiredString(data.releasePath, "releasePath"),
|
|
1405
1498
|
revision: requiredString(data.revision, "revision")
|
|
1406
|
-
}))
|
|
1499
|
+
}), {waitingFor: "the retained release generations to settle"})
|
|
1407
1500
|
}
|
|
1408
1501
|
|
|
1409
1502
|
if (commandName === "status") {
|
|
1410
|
-
|
|
1503
|
+
const status = this.status({includeLogs: statusIncludeLogs(data.includeLogs)})
|
|
1504
|
+
let guardian
|
|
1505
|
+
|
|
1506
|
+
try {
|
|
1507
|
+
const diagnostics = await this.guardian?.guardianStatus()
|
|
1508
|
+
|
|
1509
|
+
guardian = diagnostics ? {guardianPid: diagnostics.guardianPid, ownerMutation: diagnostics.ownerMutation, version: diagnostics.version} : {guardianPid: null, ownerMutation: null, version: null}
|
|
1510
|
+
} catch (error) {
|
|
1511
|
+
guardian = {error: error instanceof Error ? error.message : String(error), guardianPid: this.guardianIdentity?.pid ?? null, ownerMutation: null, version: null}
|
|
1512
|
+
}
|
|
1513
|
+
|
|
1514
|
+
return {...status, guardian}
|
|
1411
1515
|
}
|
|
1412
1516
|
|
|
1413
1517
|
if (commandName === "events") {
|
|
@@ -1418,18 +1522,18 @@ export default class RollbridgeDaemon {
|
|
|
1418
1522
|
return await this.executeOwnerMutation("stop", async () => {
|
|
1419
1523
|
await this.stopRelease(stringOrUndefined(data.releaseId))
|
|
1420
1524
|
return this.status()
|
|
1421
|
-
})
|
|
1525
|
+
}, {waitingFor: "the release's managed processes to stop"})
|
|
1422
1526
|
}
|
|
1423
1527
|
|
|
1424
1528
|
if (commandName === "restart") {
|
|
1425
1529
|
return await this.executeOwnerMutation("restart", async () => await this.restartProcesses({
|
|
1426
1530
|
policy: stringOrUndefined(data.policy),
|
|
1427
1531
|
processId: stringOrUndefined(data.processId)
|
|
1428
|
-
}))
|
|
1532
|
+
}), {waitingFor: "the managed process to stop and start again"})
|
|
1429
1533
|
}
|
|
1430
1534
|
|
|
1431
1535
|
if (commandName === "rollback") {
|
|
1432
|
-
return await this.executeOwnerMutation("rollback", async () => await this.rollback({releaseId: stringOrUndefined(data.releaseId)}))
|
|
1536
|
+
return await this.executeOwnerMutation("rollback", async () => await this.rollback({releaseId: stringOrUndefined(data.releaseId)}), {waitingFor: "the previous release to activate"})
|
|
1433
1537
|
}
|
|
1434
1538
|
|
|
1435
1539
|
if (commandName === "shutdown") {
|
|
@@ -1493,11 +1597,13 @@ export default class RollbridgeDaemon {
|
|
|
1493
1597
|
* Runs one owner mutation under the guardian's prepare/commit exclusion fence.
|
|
1494
1598
|
* @param {string} operation - Mutation diagnostic name.
|
|
1495
1599
|
* @param {() => Promise<Record<string, JsonValue>>} callback - Mutating operation.
|
|
1600
|
+
* @param {{waitingFor?: string}} [options] - What this mutation blocks on, so a refused
|
|
1601
|
+
* later mutation names the real blocker instead of just "in progress".
|
|
1496
1602
|
* @returns {Promise<Record<string, JsonValue>>} Operation result.
|
|
1497
1603
|
*/
|
|
1498
|
-
async executeOwnerMutation(operation, callback) {
|
|
1604
|
+
async executeOwnerMutation(operation, callback, {waitingFor} = {}) {
|
|
1499
1605
|
if (!this.guardian) return await callback()
|
|
1500
|
-
const mutationId = await this.guardian.beginOwnerMutation(operation)
|
|
1606
|
+
const mutationId = await this.guardian.beginOwnerMutation(operation, {ownerPid: process.pid, waitingFor})
|
|
1501
1607
|
let result
|
|
1502
1608
|
let operationError
|
|
1503
1609
|
|
|
@@ -1633,6 +1739,46 @@ export default class RollbridgeDaemon {
|
|
|
1633
1739
|
await this.updateGenerationTransition("restoring_committed")
|
|
1634
1740
|
return await this.resumeGenerationTransition()
|
|
1635
1741
|
}
|
|
1742
|
+
await this.escalateWedgedGenerations(nextConfig, newReleaseId)
|
|
1743
|
+
// A release id that is still retained by a generation holding live work cannot be started
|
|
1744
|
+
// a second time: the candidate's processes re-register the guardian keys that the retained
|
|
1745
|
+
// generation's own processes still hold, so the deploy aborts partway through `start()`
|
|
1746
|
+
// with a bare `Guardian provenance mismatch` — after the caller has committed to this
|
|
1747
|
+
// revision and, for a restore, after the deployment pointer has already moved. Refusing
|
|
1748
|
+
// from the retained record before any process starts keeps the incumbent serving, names the
|
|
1749
|
+
// release the guardian reports as active, and leaves the retained generation intact for
|
|
1750
|
+
// `rollbridge status` to reconcile against. Journaled transition resume above is exempt:
|
|
1751
|
+
// it re-drives the transaction that already owns this release id.
|
|
1752
|
+
//
|
|
1753
|
+
// The escalation above runs first on purpose: it is what clears the residue that made this
|
|
1754
|
+
// guard refuse the measured 2026-11-15 and 2026-11-16 deploys. A wedge that escalation can
|
|
1755
|
+
// prove is kill-safe stops holding its keys and the handoff proceeds; a generation that
|
|
1756
|
+
// still holds live work is not killed and is named here instead of aborting mid-`start()`.
|
|
1757
|
+
const heldGeneration = this.retainedGenerationHoldingLiveWork(newReleaseId)
|
|
1758
|
+
|
|
1759
|
+
if (heldGeneration) {
|
|
1760
|
+
const activeReleaseId = this.activeRelease?.releaseId ?? null
|
|
1761
|
+
|
|
1762
|
+
this.logger("deploy refused: retained generation still holds live processes", {
|
|
1763
|
+
activeReleaseId,
|
|
1764
|
+
heldProcesses: heldGeneration.processes,
|
|
1765
|
+
releaseId: newReleaseId,
|
|
1766
|
+
retainedState: heldGeneration.state
|
|
1767
|
+
})
|
|
1768
|
+
|
|
1769
|
+
throw new Error(`Deploy refused before activation: release "${newReleaseId}" is retained in state ${heldGeneration.state} while ${heldGeneration.count} process${heldGeneration.count === 1 ? "" : "es"} of that generation still hold live work (${heldGeneration.hold.join("; ")}). Activating that release id again would re-register those processes under a new provenance and abort with a guardian provenance mismatch. The guardian's active release is ${activeReleaseId ? `"${activeReleaseId}"` : "none"}; deploy a new release id or reconcile the deployment pointer with \`rollbridge status\`.`)
|
|
1770
|
+
}
|
|
1771
|
+
const admission = this.evaluateGenerationAdmission(newReleaseId, nextConfig)
|
|
1772
|
+
|
|
1773
|
+
if (!admission.allowed) {
|
|
1774
|
+
this.logger("deploy refused by draining-generation bound", {
|
|
1775
|
+
candidateReleaseId: newReleaseId,
|
|
1776
|
+
draining: admission.draining.map((group) => ({releaseId: group.releaseId, status: group.status})),
|
|
1777
|
+
limit: nextConfig.releaseRetention.maxConcurrentDrainingReleaseGroups,
|
|
1778
|
+
wouldBeDrainingCount: admission.wouldBeDrainingCount
|
|
1779
|
+
})
|
|
1780
|
+
throw new Error(/** @type {string} */ (admission.reason))
|
|
1781
|
+
}
|
|
1636
1782
|
const release = new ReleaseGroup({
|
|
1637
1783
|
config: nextConfig,
|
|
1638
1784
|
logger: this.logger,
|
|
@@ -1689,6 +1835,79 @@ export default class RollbridgeDaemon {
|
|
|
1689
1835
|
return await this.resumeGenerationTransition()
|
|
1690
1836
|
}
|
|
1691
1837
|
|
|
1838
|
+
/**
|
|
1839
|
+
* Applies the draining-generation bound before a handoff.
|
|
1840
|
+
*
|
|
1841
|
+
* Without this, every deploy retires one more generation and the backlog grows
|
|
1842
|
+
* monotonically whenever a drain outlasts the deploy cadence; a consumer that checks the
|
|
1843
|
+
* count after activation reports the growth it cannot undo. Refusing here names the
|
|
1844
|
+
* draining groups and whether each is still progressing or wedged, before any process of
|
|
1845
|
+
* the candidate is started.
|
|
1846
|
+
* @param {string} candidateReleaseId - Release the handoff would activate.
|
|
1847
|
+
* @param {import("./config.js").RollbridgeConfig} config - Config governing the handoff.
|
|
1848
|
+
* @returns {import("./drain-policy.js").GenerationAdmission} Admission decision and the groups behind it.
|
|
1849
|
+
*/
|
|
1850
|
+
evaluateGenerationAdmission(candidateReleaseId, config) {
|
|
1851
|
+
const draining = [...this.releases.values()]
|
|
1852
|
+
.filter((release) => release.state === "draining")
|
|
1853
|
+
.map((release) => {
|
|
1854
|
+
const drain = release.drainStatus()
|
|
1855
|
+
|
|
1856
|
+
return {reason: drain?.reason ?? "no liveness observation recorded", releaseId: release.releaseId, status: drain?.status ?? "unproven"}
|
|
1857
|
+
})
|
|
1858
|
+
|
|
1859
|
+
return admitGenerationRelease({
|
|
1860
|
+
candidateReleaseId,
|
|
1861
|
+
draining,
|
|
1862
|
+
incumbentWillDrain: this.activeRelease !== undefined,
|
|
1863
|
+
maxConcurrentDrainingReleaseGroups: config.releaseRetention.maxConcurrentDrainingReleaseGroups
|
|
1864
|
+
})
|
|
1865
|
+
}
|
|
1866
|
+
|
|
1867
|
+
/**
|
|
1868
|
+
* Reports whether a retained release group still holds live work that a re-activation would collide with.
|
|
1869
|
+
*
|
|
1870
|
+
* `stopped` is not the same claim as `drained`: the guardian's provenance fence only lets a
|
|
1871
|
+
* process key be re-registered once the record that holds it is gone. A generation recorded
|
|
1872
|
+
* as drained while its process group is still alive — the shape an indefinite worker drain
|
|
1873
|
+
* produces — still holds its keys, so its release id is not available for a restore.
|
|
1874
|
+
* @param {string} releaseId - Release id a handoff would activate.
|
|
1875
|
+
* @returns {HeldGenerationHold | undefined} The live hold, or undefined when the id is free.
|
|
1876
|
+
*/
|
|
1877
|
+
retainedGenerationHoldingLiveWork(releaseId) {
|
|
1878
|
+
const retained = this.releases.get(releaseId)
|
|
1879
|
+
|
|
1880
|
+
if (!retained || retained === this.activeRelease) return undefined
|
|
1881
|
+
|
|
1882
|
+
return heldGenerationHold(retained)
|
|
1883
|
+
}
|
|
1884
|
+
|
|
1885
|
+
/**
|
|
1886
|
+
* Gives every retiring generation the escalation decision its own retirement left open,
|
|
1887
|
+
* before the handoff is judged.
|
|
1888
|
+
*
|
|
1889
|
+
* A deploy is the event that makes a wedged generation expensive: until something tries to
|
|
1890
|
+
* activate a successor, a retired release holding a live process group costs nothing
|
|
1891
|
+
* observable. Acting here means the observation that proves the wedge and the action on it
|
|
1892
|
+
* come from the same read the admission bound already performs, so no sampler has to run
|
|
1893
|
+
* alongside the daemon. Generations that already recorded `release drained` are included:
|
|
1894
|
+
* their retirement finished, and the process group they left behind is the residue that
|
|
1895
|
+
* blocked the measured 2026-11-15 deploys.
|
|
1896
|
+
* @param {import("./config.js").RollbridgeConfig} config - Config governing the handoff.
|
|
1897
|
+
* @param {string} candidateReleaseId - Release the handoff would activate.
|
|
1898
|
+
* @returns {Promise<void>} Resolves once every retiring generation has been decided.
|
|
1899
|
+
*/
|
|
1900
|
+
async escalateWedgedGenerations(config, candidateReleaseId) {
|
|
1901
|
+
if (!config.releaseRetention.escalateWedgedGenerations) return
|
|
1902
|
+
|
|
1903
|
+
for (const release of this.releases.values()) {
|
|
1904
|
+
if (release.releaseId === candidateReleaseId) continue
|
|
1905
|
+
if (release.state !== "draining" && release.state !== "stopped" && release.state !== "failed") continue
|
|
1906
|
+
|
|
1907
|
+
await release.escalateWedgedGeneration({reason: `deploy of ${candidateReleaseId}`})
|
|
1908
|
+
}
|
|
1909
|
+
}
|
|
1910
|
+
|
|
1692
1911
|
/**
|
|
1693
1912
|
* Continues one exact durable generation transition without an internal retry loop.
|
|
1694
1913
|
* @returns {Promise<Record<string, JsonValue>>} Deploy result after commit.
|
|
@@ -1710,10 +1929,9 @@ export default class RollbridgeDaemon {
|
|
|
1710
1929
|
let retirementFailure = activationLifecycle ? undefined : previousRelease?.retirementError
|
|
1711
1930
|
|
|
1712
1931
|
if (transition.phase === "candidate_ready") {
|
|
1713
|
-
//
|
|
1714
|
-
//
|
|
1715
|
-
//
|
|
1716
|
-
// carries traffic (STOE-2269).
|
|
1932
|
+
// The candidate has already started and passed its health gates. Keep the
|
|
1933
|
+
// incumbent authoritative until the candidate is activated and traffic is
|
|
1934
|
+
// switched; retirement is deliberately launched from committed_pending.
|
|
1717
1935
|
await this.updateGenerationTransition("activating_candidate")
|
|
1718
1936
|
}
|
|
1719
1937
|
|
|
@@ -1788,6 +2006,22 @@ export default class RollbridgeDaemon {
|
|
|
1788
2006
|
|
|
1789
2007
|
if (transition.phase === "committed_pending") {
|
|
1790
2008
|
await this.refreshServiceDefinitions(release)
|
|
2009
|
+
if (previousRelease?.state === "active") {
|
|
2010
|
+
if (this.activeRelease !== release) {
|
|
2011
|
+
throw new Error(`Cannot retire incumbent release ${previousRelease.releaseId} before candidate release ${release.releaseId} is authoritative.`)
|
|
2012
|
+
}
|
|
2013
|
+
const retirementConfig = previousRelease.preserveConfigOnRetirement ? previousRelease.config : release.config
|
|
2014
|
+
|
|
2015
|
+
// Traffic is already on the healthy candidate. Retirement must not hold
|
|
2016
|
+
// the deploy mutation or its owner fence; its own bounded drain reports
|
|
2017
|
+
// failures and escalates any process that misses the retirement budget.
|
|
2018
|
+
void previousRelease.beginRetirement(retirementConfig)
|
|
2019
|
+
.catch((error) => {
|
|
2020
|
+
retirementFailure = previousRelease.retirementError ?? (error instanceof Error ? error.message : String(error))
|
|
2021
|
+
this.logger("release retirement quiescence failed", {error: retirementFailure, releaseId: previousRelease.releaseId})
|
|
2022
|
+
})
|
|
2023
|
+
.then(() => this.drainAndPrune(previousRelease, retirementConfig))
|
|
2024
|
+
}
|
|
1791
2025
|
await this.checkpointGenerationTransition()
|
|
1792
2026
|
try {
|
|
1793
2027
|
await this.replaceSingletons(release)
|
|
@@ -1795,29 +2029,12 @@ export default class RollbridgeDaemon {
|
|
|
1795
2029
|
await this.failGenerationTransition(error instanceof Error ? error.message : String(error))
|
|
1796
2030
|
throw error
|
|
1797
2031
|
}
|
|
1798
|
-
if (previousRelease) {
|
|
1799
|
-
const retirementConfig = previousRelease.preserveConfigOnRetirement ? previousRelease.config : release.config
|
|
1800
|
-
|
|
1801
|
-
// Traffic is already on the candidate and singleton ownership has been
|
|
1802
|
-
// transferred. Retirement is deliberately detached so a slow old generation
|
|
1803
|
-
// cannot hold the committed deploy open or race guardian-owned replacement.
|
|
1804
|
-
void this.drainAndPrune(previousRelease, retirementConfig)
|
|
1805
|
-
}
|
|
1806
2032
|
transition.phase = "committed"
|
|
1807
2033
|
transition.error = undefined
|
|
1808
2034
|
transition.journalRevision = (transition.journalRevision ?? 0) + 1
|
|
1809
2035
|
transition.updatedAt = new Date().toISOString()
|
|
1810
2036
|
this.pruneStoppedReleases()
|
|
1811
2037
|
await this.checkpointGenerationTransition()
|
|
1812
|
-
if (previousRelease) {
|
|
1813
|
-
const retirementConfig = !activationLifecycle && previousRelease.preserveConfigOnRetirement ? previousRelease.config : release.config
|
|
1814
|
-
|
|
1815
|
-
// Traffic is already on the healthy candidate. Retirement is deliberately detached
|
|
1816
|
-
// from the owner mutation so a slow quiet/drain hook can never hold the deploy fence
|
|
1817
|
-
// or make the active successor unavailable. drainAndStop applies the bounded force
|
|
1818
|
-
// deadline and records any retirement failure on the old generation.
|
|
1819
|
-
void this.drainAndPrune(previousRelease, retirementConfig)
|
|
1820
|
-
}
|
|
1821
2038
|
}
|
|
1822
2039
|
|
|
1823
2040
|
return {
|
|
@@ -1847,6 +2064,8 @@ export default class RollbridgeDaemon {
|
|
|
1847
2064
|
if (!candidate) throw new Error(`Generation transition candidate ${transition.candidateReleaseId} is not retained`)
|
|
1848
2065
|
if (!previous) throw new Error(`Generation transition previous release ${transition.previousReleaseId} is not retained`)
|
|
1849
2066
|
if (this.activeRelease !== previous) throw new Error(`Previous release ${previous.releaseId} is not the authoritative proxy target`)
|
|
2067
|
+
/** @type {Promise<void> | undefined} */
|
|
2068
|
+
let candidateDrain
|
|
1850
2069
|
|
|
1851
2070
|
if (transition.phase === "activating_candidate") {
|
|
1852
2071
|
if (!transition.error && !transition.activationError) throw new Error("Candidate activation has no recorded failure to compensate")
|
|
@@ -1878,6 +2097,15 @@ export default class RollbridgeDaemon {
|
|
|
1878
2097
|
transition.compensationError = undefined
|
|
1879
2098
|
await this.updateGenerationTransition("restoring_previous")
|
|
1880
2099
|
this.logger("release generation compensation candidate retired", {releaseId: candidate.releaseId})
|
|
2100
|
+
|
|
2101
|
+
// The candidate's bounded drain starts here, before the restoration attempt. The only other
|
|
2102
|
+
// compensation drain runs after the success path, so when incumbent restoration throws the
|
|
2103
|
+
// candidate is left `draining` with a live process: measured 2026-11-17, candidate
|
|
2104
|
+
// `20261117063703` still held `web` pid 3201928 at 07:30Z after its compensation threw at
|
|
2105
|
+
// 06:42:31Z, and reached `release drained` only when a later compensation reached the
|
|
2106
|
+
// success path at 08:47:00Z. A retained generation holding live work is what refuses every
|
|
2107
|
+
// later deploy, and a failed candidate is never a restoration target.
|
|
2108
|
+
candidateDrain = this.drainAndPrune(candidate, candidate.config)
|
|
1881
2109
|
}
|
|
1882
2110
|
|
|
1883
2111
|
if (transition.phase === "restoring_previous") {
|
|
@@ -1889,10 +2117,12 @@ export default class RollbridgeDaemon {
|
|
|
1889
2117
|
|
|
1890
2118
|
transition.compensationError = failure
|
|
1891
2119
|
await this.failGenerationTransition(`Incumbent ${previous.releaseId} restoration failed: ${failure}`)
|
|
2120
|
+
await previous.stopProxyTraffic()
|
|
1892
2121
|
this.logger("release generation compensation restoration failed", {
|
|
1893
2122
|
activationError: transition.activationError,
|
|
1894
2123
|
error: failure,
|
|
1895
|
-
releaseId: previous.releaseId
|
|
2124
|
+
releaseId: previous.releaseId,
|
|
2125
|
+
traffic: "unavailable"
|
|
1896
2126
|
})
|
|
1897
2127
|
throw new Error(`incumbent ${previous.releaseId} restoration failed: ${failure}`, {cause: error})
|
|
1898
2128
|
}
|
|
@@ -1931,7 +2161,7 @@ export default class RollbridgeDaemon {
|
|
|
1931
2161
|
this.logger("release generation compensation checkpoint failed", {error: failure, releaseId: candidate.releaseId})
|
|
1932
2162
|
throw new Error(`compensation checkpoint clear failed: ${failure}`, {cause: error})
|
|
1933
2163
|
}
|
|
1934
|
-
void this.drainAndPrune(candidate, candidate.config)
|
|
2164
|
+
if (candidateDrain === undefined) void this.drainAndPrune(candidate, candidate.config)
|
|
1935
2165
|
this.logger("release generation compensation completed", result)
|
|
1936
2166
|
return result
|
|
1937
2167
|
}
|
|
@@ -1955,6 +2185,19 @@ export default class RollbridgeDaemon {
|
|
|
1955
2185
|
recoveryStatus: "already_recovered"
|
|
1956
2186
|
}
|
|
1957
2187
|
}
|
|
2188
|
+
// A committed journal describes the transition that already published the current
|
|
2189
|
+
// incumbent, not an in-flight transition to recover. When the recovery fence names
|
|
2190
|
+
// that live incumbent, the failed deploy never journaled its own transition, so the
|
|
2191
|
+
// only fence that describes the authoritative release is the completed one: recovery
|
|
2192
|
+
// is already done. In-flight phases keep the stale-fence guard failing closed.
|
|
2193
|
+
if (transition.phase === "committed" && transition.candidateReleaseId === identity.previousReleaseId && this.activeRelease?.releaseId === identity.previousReleaseId && this.releases.get(transition.candidateReleaseId) === this.activeRelease) {
|
|
2194
|
+
return {
|
|
2195
|
+
activeReleaseId: this.activeRelease.releaseId,
|
|
2196
|
+
candidateReleaseId: identity.releaseId,
|
|
2197
|
+
previousReleaseId: identity.previousReleaseId,
|
|
2198
|
+
recoveryStatus: "already_recovered"
|
|
2199
|
+
}
|
|
2200
|
+
}
|
|
1958
2201
|
if (transition.previousReleaseId !== identity.previousReleaseId) {
|
|
1959
2202
|
throw new Error(`Generation transition previous release is ${transition.previousReleaseId}; refusing stale recovery for ${identity.previousReleaseId}`)
|
|
1960
2203
|
}
|
|
@@ -2401,11 +2644,38 @@ export default class RollbridgeDaemon {
|
|
|
2401
2644
|
throw new Error(`Release "${target.releaseId}" is already active.`)
|
|
2402
2645
|
}
|
|
2403
2646
|
|
|
2647
|
+
// A generation recorded as `stopped` whose processes are still alive is the shape a
|
|
2648
|
+
// restore cannot honour: re-deploying that release id re-registers its guardian process
|
|
2649
|
+
// keys under a new provenance while the old processes still hold them, and the guardian
|
|
2650
|
+
// answers with a bare provenance mismatch. Use the guardian's own fence here so the
|
|
2651
|
+
// refusal is decided before anything is touched, and name the divergence — including a
|
|
2652
|
+
// process group that outlived the tracked process and is invisible as a pid — together
|
|
2653
|
+
// with the release the guardian actually reports as active, so the deployment tool can
|
|
2654
|
+
// reconcile its own current pointer instead of restoring a generation that already drained.
|
|
2655
|
+
// A target that is still draining is not refused here: stopping it is what makes its id
|
|
2656
|
+
// reusable, and the deploy below applies the same fence after that stop.
|
|
2657
|
+
const heldGeneration = target.state === "stopped" ? heldGenerationHold(target) : undefined
|
|
2658
|
+
|
|
2659
|
+
if (heldGeneration) {
|
|
2660
|
+
const activeReleaseId = this.activeRelease?.releaseId ?? null
|
|
2661
|
+
|
|
2662
|
+
this.logger("rollback refused: drained generation still holds live processes", {
|
|
2663
|
+
activeReleaseId,
|
|
2664
|
+
heldProcesses: heldGeneration.processes,
|
|
2665
|
+
releaseId: target.releaseId,
|
|
2666
|
+
revision: target.revision
|
|
2667
|
+
})
|
|
2668
|
+
|
|
2669
|
+
throw new Error(`Release "${target.releaseId}" is recorded as drained while ${heldGeneration.count} process${heldGeneration.count === 1 ? "" : "es"} of that generation still hold live work (${heldGeneration.hold.join("; ")}). Restoring it would re-register those processes under a new provenance and fail with a guardian provenance mismatch. The guardian's active release is ${activeReleaseId ? `"${activeReleaseId}"` : "none"}; reconcile the deployment pointer with \`rollbridge status\` before restoring.`)
|
|
2670
|
+
}
|
|
2671
|
+
|
|
2404
2672
|
// The target may still be draining a prior deploy (live processes). Stop it before the
|
|
2405
2673
|
// deploy below re-uses its id in this.releases, otherwise the still-running instance
|
|
2406
2674
|
// would be dropped from status/pruning/shutdown and could be orphaned.
|
|
2675
|
+
// Rollback is an owner mutation too, so this stop carries the same retirement bound as
|
|
2676
|
+
// `stop`: a process the supervisor must not kill must not be able to hold the fence.
|
|
2407
2677
|
if (target.state !== "stopped" && target.state !== "failed") {
|
|
2408
|
-
await target.stop()
|
|
2678
|
+
await target.stop({retirementTimeoutMs: this.config.proxy.nonBlockingDrainTimeoutMs})
|
|
2409
2679
|
}
|
|
2410
2680
|
|
|
2411
2681
|
this.logger("rollback starting", {releaseId: target.releaseId, releasePath: target.releasePath})
|
|
@@ -2626,7 +2896,7 @@ export default class RollbridgeDaemon {
|
|
|
2626
2896
|
|
|
2627
2897
|
/**
|
|
2628
2898
|
* @param {string | undefined} releaseId - Release id, or active release when omitted.
|
|
2629
|
-
* @returns {Promise<void>} Resolves when stopped.
|
|
2899
|
+
* @returns {Promise<void>} Resolves when stopped, or when retirement stops waiting.
|
|
2630
2900
|
*/
|
|
2631
2901
|
async stopRelease(releaseId) {
|
|
2632
2902
|
this.assertNoUnresolvedGenerationTransition("stop")
|
|
@@ -2635,7 +2905,10 @@ export default class RollbridgeDaemon {
|
|
|
2635
2905
|
if (!release) throw new Error(`Release not found: ${releaseId || "active"}`)
|
|
2636
2906
|
if (release === this.activeRelease) this.activeRelease = undefined
|
|
2637
2907
|
|
|
2638
|
-
|
|
2908
|
+
// This command runs inside the owner mutation, so an unbudgeted wait for a process the
|
|
2909
|
+
// supervisor must not kill holds the fence for the whole drain and every later deploy is
|
|
2910
|
+
// refused. The retirement bound is the same one the deploy path already uses.
|
|
2911
|
+
await release.stop({retirementTimeoutMs: this.config.proxy.nonBlockingDrainTimeoutMs})
|
|
2639
2912
|
this.logger("release stopped", {releaseId: release.releaseId})
|
|
2640
2913
|
this.pruneStoppedReleases()
|
|
2641
2914
|
this.persistState()
|
|
@@ -2840,11 +3113,16 @@ export default class RollbridgeDaemon {
|
|
|
2840
3113
|
await this.guardian.retireOwner()
|
|
2841
3114
|
this.guardian.disconnect()
|
|
2842
3115
|
} else {
|
|
3116
|
+
// Without a guardian nobody else can retire these processes, so an unbounded stop would
|
|
3117
|
+
// leave the generation draining forever — the state that made the measured 2026-11-17
|
|
3118
|
+
// rollback refuse. The same retirement budget terminalizes it and names the residue.
|
|
3119
|
+
const retirementTimeoutMs = this.config.proxy.nonBlockingDrainTimeoutMs
|
|
3120
|
+
|
|
2843
3121
|
void Promise.allSettled([
|
|
2844
|
-
...[...this.services.values()].map((processInstance) => processInstance.stop()),
|
|
2845
|
-
...[...this.singletons.values()].map((processInstance) => processInstance.stop()),
|
|
2846
|
-
...[...this.startingReleases].map((release) => release.stop()),
|
|
2847
|
-
...[...this.releases.values()].map((release) => release.stop())
|
|
3122
|
+
...[...this.services.values()].map((processInstance) => processInstance.stop({retirementTimeoutMs})),
|
|
3123
|
+
...[...this.singletons.values()].map((processInstance) => processInstance.stop({retirementTimeoutMs})),
|
|
3124
|
+
...[...this.startingReleases].map((release) => release.stop({retirementTimeoutMs})),
|
|
3125
|
+
...[...this.releases.values()].map((release) => release.stop({retirementTimeoutMs}))
|
|
2848
3126
|
])
|
|
2849
3127
|
}
|
|
2850
3128
|
this.logger("external owner retired", {attestation, status: "draining"})
|
|
@@ -2878,12 +3156,19 @@ export default class RollbridgeDaemon {
|
|
|
2878
3156
|
}
|
|
2879
3157
|
|
|
2880
3158
|
await captureShutdownError(cleanupErrors, "proxy close", async () => this.proxy.close())
|
|
3159
|
+
// Shutdown must return even when a process refuses to exit. The bound is the
|
|
3160
|
+
// `proxy.nonBlockingDrainTimeoutMs` an explicit `rollbridge stop` already uses, and it means
|
|
3161
|
+
// "stop waiting", never "SIGKILL": a process whose graceful window is `"indefinite"` is left
|
|
3162
|
+
// draining rather than killed. Left unbudgeted, that stop keeps the daemon alive after its
|
|
3163
|
+
// control socket is already unlinked — alive, unreachable, and still holding the generation it
|
|
3164
|
+
// stopped: the wedged daemon STOE-2275 measured behind 4,800 `connect ECONNREFUSED` proxy errors.
|
|
3165
|
+
const retirementTimeoutMs = this.config.proxy.nonBlockingDrainTimeoutMs
|
|
2881
3166
|
const dependentStopResults = await Promise.allSettled([
|
|
2882
|
-
...[...this.singletons.values()].map((processInstance) => processInstance.stop()),
|
|
2883
|
-
...[...this.startingReleases].map((release) => release.stop()),
|
|
2884
|
-
...[...this.releases.values()].map((release) => release.stop())
|
|
3167
|
+
...[...this.singletons.values()].map((processInstance) => processInstance.stop({retirementTimeoutMs})),
|
|
3168
|
+
...[...this.startingReleases].map((release) => release.stop({retirementTimeoutMs})),
|
|
3169
|
+
...[...this.releases.values()].map((release) => release.stop({retirementTimeoutMs}))
|
|
2885
3170
|
])
|
|
2886
|
-
const serviceStopResults = await Promise.allSettled([...this.services.values()].map((processInstance) => processInstance.stop()))
|
|
3171
|
+
const serviceStopResults = await Promise.allSettled([...this.services.values()].map((processInstance) => processInstance.stop({retirementTimeoutMs})))
|
|
2887
3172
|
const stopResults = [...dependentStopResults, ...serviceStopResults]
|
|
2888
3173
|
const guardian = this.guardian
|
|
2889
3174
|
if (guardian) await captureShutdownError(cleanupErrors, "process guardian shutdown", () => guardian.shutdown())
|
|
@@ -3067,6 +3352,42 @@ function statusIncludeLogs(value) {
|
|
|
3067
3352
|
return value
|
|
3068
3353
|
}
|
|
3069
3354
|
|
|
3355
|
+
/**
|
|
3356
|
+
* Names the live work a retained release group still holds, using the guardian's own fence.
|
|
3357
|
+
*
|
|
3358
|
+
* A process counts as holding work when the guardian would refuse to hand its key to a new
|
|
3359
|
+
* provenance: a running or stopping process, or a record already `stopped` whose process group
|
|
3360
|
+
* outlived it and is invisible as a pid.
|
|
3361
|
+
* @param {ReleaseGroup} release - Retained generation to inspect.
|
|
3362
|
+
* @returns {HeldGenerationHold | undefined} The hold and its log payload, or undefined when every process is gone.
|
|
3363
|
+
*/
|
|
3364
|
+
function heldGenerationHold(release) {
|
|
3365
|
+
const heldProcesses = release.status().processes.filter((processStatus) => !childRecordIsGone(processStatus))
|
|
3366
|
+
|
|
3367
|
+
if (heldProcesses.length === 0) return undefined
|
|
3368
|
+
|
|
3369
|
+
return {
|
|
3370
|
+
count: heldProcesses.length,
|
|
3371
|
+
hold: heldProcesses.map((processStatus) => {
|
|
3372
|
+
if (processStatus.heldProcessGroup) {
|
|
3373
|
+
const {liveMemberCount, pgid} = processStatus.heldProcessGroup
|
|
3374
|
+
const members = liveMemberCount === undefined ? "" : ` with ${liveMemberCount} live member(s)`
|
|
3375
|
+
|
|
3376
|
+
return `${processStatus.id}: process group ${pgid}${members} still alive after the tracked process exited`
|
|
3377
|
+
}
|
|
3378
|
+
|
|
3379
|
+
return `${processStatus.id}: state ${processStatus.state}, pid ${processStatus.pid ?? "unknown"}`
|
|
3380
|
+
}),
|
|
3381
|
+
processes: heldProcesses.map((processStatus) => ({
|
|
3382
|
+
heldProcessGroupPgid: processStatus.heldProcessGroup?.pgid ?? null,
|
|
3383
|
+
pid: processStatus.pid ?? null,
|
|
3384
|
+
processId: processStatus.id,
|
|
3385
|
+
state: processStatus.state
|
|
3386
|
+
})),
|
|
3387
|
+
state: release.state
|
|
3388
|
+
}
|
|
3389
|
+
}
|
|
3390
|
+
|
|
3070
3391
|
/**
|
|
3071
3392
|
* @param {PrivateOwnerState} transfer - Authenticated private incumbent state.
|
|
3072
3393
|
* @param {GenerationTransition} transition - Unresolved generation transition.
|
|
@@ -3127,6 +3448,15 @@ export function isLegacyGuardianPrepareDiagnostic(diagnostic) {
|
|
|
3127
3448
|
diagnostic === "Unknown guardian command: prepare-owner-replacement"
|
|
3128
3449
|
}
|
|
3129
3450
|
|
|
3451
|
+
/**
|
|
3452
|
+
* Reports the capability diagnostics that identify a guardian predating daemon recovery.
|
|
3453
|
+
* @param {string} diagnostic - Guardian error message.
|
|
3454
|
+
* @returns {boolean} Whether the guardian predates the daemon recovery capability.
|
|
3455
|
+
*/
|
|
3456
|
+
function isGuardianDaemonRecoveryDiagnostic(diagnostic) {
|
|
3457
|
+
return diagnostic === "Guardian capabilities requires a process key" || diagnostic === "Unknown guardian command: capabilities"
|
|
3458
|
+
}
|
|
3459
|
+
|
|
3130
3460
|
/**
|
|
3131
3461
|
* @param {DaemonStatus} snapshot - Serialized owner process snapshot.
|
|
3132
3462
|
* @param {Record<string, string>} [singletonReleaseIds] - Exact singleton owner releases.
|
|
@@ -3449,7 +3779,7 @@ function requiredNonNegativeInteger(value, key) {
|
|
|
3449
3779
|
/**
|
|
3450
3780
|
* Selects stopped releases to prune by the retention policy, keeping the most recent.
|
|
3451
3781
|
* @param {PrunableRelease[]} releases - Status of all tracked releases, in deploy order (oldest first).
|
|
3452
|
-
* @param {
|
|
3782
|
+
* @param {{keep: number, maxAgeMs: number}} policy - Retention policy; draining generations are never pruned here.
|
|
3453
3783
|
* @param {number} now - Current epoch milliseconds.
|
|
3454
3784
|
* @returns {string[]} Release ids to remove.
|
|
3455
3785
|
*/
|
|
@@ -3474,7 +3804,7 @@ export function releasesToPrune(releases, policy, now) {
|
|
|
3474
3804
|
}
|
|
3475
3805
|
|
|
3476
3806
|
/**
|
|
3477
|
-
* @typedef {{alive: boolean, application?: string, activeReleaseId?: string | null, proxy?: {host: string, port: number}}} ControlSocketInspection
|
|
3807
|
+
* @typedef {{alive: boolean, application?: string, activeReleaseId?: string | null, guardian?: {ownerMutation: Record<string, JsonValue> | null, version: string | null}, proxy?: {host: string, port: number}}} ControlSocketInspection
|
|
3478
3808
|
*/
|
|
3479
3809
|
|
|
3480
3810
|
/**
|
|
@@ -3549,7 +3879,7 @@ export async function inspectControlSocket(socketPath, timeoutMs = 1000) {
|
|
|
3549
3879
|
/**
|
|
3550
3880
|
* Parses a control status response line into a Rollbridge identity, if it is one.
|
|
3551
3881
|
* @param {string} line - JSON response line.
|
|
3552
|
-
* @returns {{application: string, activeReleaseId: string | null, proxy: {host: string, port: number} | undefined} | undefined} Identity, or undefined when unrecognized.
|
|
3882
|
+
* @returns {{application: string, activeReleaseId: string | null, guardian: {ownerMutation: Record<string, JsonValue> | null, version: string | null} | undefined, proxy: {host: string, port: number} | undefined} | undefined} Identity, or undefined when unrecognized.
|
|
3553
3883
|
*/
|
|
3554
3884
|
function parseControlStatus(line) {
|
|
3555
3885
|
/** @type {JsonValue} */
|
|
@@ -3566,5 +3896,8 @@ function parseControlStatus(line) {
|
|
|
3566
3896
|
|
|
3567
3897
|
const proxy = "proxy" in parsed && parsed.proxy && typeof parsed.proxy === "object" && !Array.isArray(parsed.proxy) && typeof parsed.proxy.host === "string" && typeof parsed.proxy.port === "number" ? {host: parsed.proxy.host, port: parsed.proxy.port} : undefined
|
|
3568
3898
|
|
|
3569
|
-
|
|
3899
|
+
const guardianValue = "guardian" in parsed && parsed.guardian && typeof parsed.guardian === "object" && !Array.isArray(parsed.guardian) ? parsed.guardian : undefined
|
|
3900
|
+
const guardian = guardianValue && typeof guardianValue.version === "string" || guardianValue && guardianValue.version === null ? {ownerMutation: guardianValue.ownerMutation && typeof guardianValue.ownerMutation === "object" && !Array.isArray(guardianValue.ownerMutation) ? guardianValue.ownerMutation : null, version: guardianValue.version} : undefined
|
|
3901
|
+
|
|
3902
|
+
return {activeReleaseId: typeof parsed.activeReleaseId === "string" ? parsed.activeReleaseId : null, application: parsed.application, guardian, proxy}
|
|
3570
3903
|
}
|