rollbridge 0.1.69 → 0.1.71
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -53
- package/TODO.md +0 -12
- package/docs/cli.md +3 -128
- package/docs/config.md +6 -177
- package/docs/troubleshooting.md +6 -97
- package/docs/velocious.md +1 -1
- package/docs/workers.md +0 -11
- package/examples/tensorbuzz.com.js +2 -3
- package/package.json +1 -1
- package/src/cli.js +2 -68
- package/src/config.js +8 -347
- package/src/daemon.js +69 -370
- package/src/doctor.js +0 -74
- package/src/guardian-client.js +8 -67
- package/src/health.js +8 -99
- package/src/managed-process.js +16 -197
- package/src/process-guardian.js +21 -211
- package/src/process-memory.js +2 -62
- package/src/release-group.js +37 -427
- package/tensorbuzz.yml +1 -20
- package/test/completion.test.js +2 -5
- package/test/config-validation.test.js +5 -135
- package/test/guardian-client.test.js +4 -118
- package/test/health.test.js +1 -74
- package/test/managed-process.test.js +1 -260
- package/test/owner-recovery.test.js +28 -132
- package/test/owner-replacement.test.js +11 -172
- package/test/process-memory.test.js +1 -47
- package/test/release-group.test.js +2 -408
- package/test/rollbridge.test.js +82 -766
- package/test/shutdown-completion.test.js +0 -52
- package/test/support/process.js +1 -3
- package/changelog.d/20261025-guardian-re-register-dead-generation.md +0 -13
- package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +0 -1
- package/changelog.d/20261104-compose-service-units.md +0 -21
- package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +0 -20
- package/changelog.d/20261106-release-dependency-boot-budgets.md +0 -32
- package/changelog.d/20261109-path-filtered-builds.md +0 -11
- package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +0 -26
- package/changelog.d/20261111-contention-sensitive-daemon-specs.md +0 -27
- package/changelog.d/20261113-bounded-nonblocking-retirement.md +0 -5
- package/changelog.d/20261115-bound-draining-release-groups.md +0 -62
- package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +0 -27
- package/changelog.d/20261115-wedged-generation-escalation.md +0 -44
- package/changelog.d/20261116-bounded-explicit-stop-retirement.md +0 -30
- package/changelog.d/20261116-config-json-canonical-shape.md +0 -11
- package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +0 -29
- package/changelog.d/20261116-guardian-version-handoff.md +0 -37
- package/changelog.d/20261116-reclaim-dead-owner-mutation.md +0 -7
- package/changelog.d/20261117-activation-path-drain-deadline.md +0 -35
- package/changelog.d/20261117-stop-companion-after-retired-dependency.md +0 -7
- package/docs/compose-services.md +0 -222
- package/src/compose-unit.js +0 -725
- package/src/drain-policy.js +0 -252
- package/src/process-order.js +0 -26
- package/test/compose-config.test.js +0 -160
- package/test/compose-docker.test.js +0 -297
- package/test/compose-unit.test.js +0 -415
- package/test/drain-policy.test.js +0 -328
- package/test/fixtures/deferred-boot-app.js +0 -106
- package/test/fixtures/gated-app.js +0 -50
- package/test/tensorbuzz-config.test.js +0 -228
package/src/daemon.js
CHANGED
|
@@ -9,10 +9,8 @@ import {isDeepStrictEqual} from "node:util"
|
|
|
9
9
|
import httpProxy from "http-proxy"
|
|
10
10
|
import {loadConfig} from "./config.js"
|
|
11
11
|
import {openControlSession} from "./control-client.js"
|
|
12
|
-
import {admitGenerationRelease} from "./drain-policy.js"
|
|
13
12
|
import EventLog from "./event-log.js"
|
|
14
|
-
import GuardianClient
|
|
15
|
-
import {childRecordIsGone} from "./managed-process.js"
|
|
13
|
+
import GuardianClient from "./guardian-client.js"
|
|
16
14
|
import ReleaseGroup from "./release-group.js"
|
|
17
15
|
import {clearState, isProcessAlive, liveProcesses, readState, writeState} from "./state-store.js"
|
|
18
16
|
import {resolveGroupId, resolveUserId} from "./system-ids.js"
|
|
@@ -25,20 +23,18 @@ const STATE_PERSIST_INTERVAL_MS = 5000
|
|
|
25
23
|
* @typedef {{releaseId?: string, releasePath: string, revision?: string}} DeployArgs
|
|
26
24
|
* @typedef {{attestation?: string, releaseId: string, releasePath: string, revision: string}} BootstrapIdentity
|
|
27
25
|
* @typedef {{id: string, process: import("./managed-process.js").ManagedProcessStatus}} ProcessStatus
|
|
28
|
-
* @typedef {{disruptive: true, mode: "
|
|
29
|
-
* @typedef {"guardian-version-drift" | "legacy-protocol"} GuardianHandoffProtocol
|
|
26
|
+
* @typedef {{disruptive: true, mode: "legacy-first-upgrade", reason: string}} OwnerTransition
|
|
30
27
|
* @typedef {"candidate_ready" | "retiring_previous" | "previous_retired" | "activating_candidate" | "restoring_previous" | "retiring_failed_candidate" | "degraded_active" | "committed_pending" | "committed" | "restoring_committed"} GenerationTransitionPhase
|
|
31
28
|
* @typedef {{activationError?: string, activationLifecycle?: boolean, candidateReleaseId: string, candidateReleasePath: string, candidateRevision: string, compensationError?: string, configDigest: string, degradedIncumbent?: boolean, error?: string, journalRevision?: number, phase: GenerationTransitionPhase, previousReleaseId: string | null, startedAt: string, updatedAt: string}} GenerationTransition
|
|
32
|
-
* @typedef {{activeReleaseId: string | null, application: string, bootstrap: BootstrapIdentity | undefined, control: import("./config.js").ControlConfig, daemonPid: number, daemonRuntime: import("./daemon-runtime.js").DaemonRuntimeIdentity | undefined, generationTransition?: GenerationTransition,
|
|
29
|
+
* @typedef {{activeReleaseId: string | null, application: string, bootstrap: BootstrapIdentity | undefined, control: import("./config.js").ControlConfig, daemonPid: number, daemonRuntime: import("./daemon-runtime.js").DaemonRuntimeIdentity | undefined, generationTransition?: GenerationTransition, ownerRecovery: {configDigest: string, ready: boolean} | undefined, ownerTransition?: OwnerTransition, orphans: {id: string, pid: number, releaseId: string | null}[], proxy: {host: string, port: number | undefined, upstreamHost: string}, releaseReferences: {releaseId: string, releasePath: string}[], releases: import("./release-group.js").ReleaseStatus[], services: ProcessStatus[], singletons: ProcessStatus[]}} DaemonStatus
|
|
33
30
|
* @typedef {Omit<import("./managed-process.js").ManagedProcessStatus, "logs">} ManagedProcessStatusWithoutLogs
|
|
34
31
|
* @typedef {Omit<import("./release-group.js").ReleaseStatus, "processes"> & {processes: ManagedProcessStatusWithoutLogs[]}} ReleaseStatusWithoutLogs
|
|
35
32
|
* @typedef {Omit<ProcessStatus, "process"> & {process: ManagedProcessStatusWithoutLogs}} ProcessStatusWithoutLogs
|
|
36
33
|
* @typedef {Omit<DaemonStatus, "releases" | "services" | "singletons"> & {releases: ReleaseStatusWithoutLogs[], services: ProcessStatusWithoutLogs[], singletons: ProcessStatusWithoutLogs[]}} DaemonStatusWithoutLogs
|
|
37
|
-
* @typedef {{count: number, hold: string[], processes: {heldProcessGroupPgid: number | null, pid: number | null, processId: string, state: import("./managed-process.js").ManagedProcessState}[], state: import("./release-group.js").ReleaseState}} HeldGenerationHold
|
|
38
34
|
* @typedef {{configDigest: string, format: number, guardian: {pid?: number, socketPath: string, token: string}, reconnectGraceMs: number}} OwnerRecoveryMetadata
|
|
39
35
|
* @typedef {DaemonStatus & {recovery: OwnerRecoveryMetadata, serviceReleaseIds?: Record<string, string>, singletonReleaseIds?: Record<string, string>}} OwnerRecoverySnapshot
|
|
40
36
|
* @typedef {{authority: JsonValue, config: import("./config.js").RollbridgeConfig, listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>, listenerSourceId?: string, recovery?: {command: JsonValue, reconnectGraceMs: number, startupTimeoutMs: number}, releaseConfigs?: Record<string, import("./config.js").RollbridgeConfig>, retiredListenerHandoff?: number, serviceReleaseIds?: Record<string, string>, singletonReleaseIds?: Record<string, string>, snapshot: OwnerRecoverySnapshot}} PrivateOwnerState
|
|
41
|
-
* @typedef {{boundaryCrossed: boolean,
|
|
37
|
+
* @typedef {{boundaryCrossed: boolean, incumbentControl?: Awaited<ReturnType<typeof openControlSession>>, incumbentStartTime: string, legacyGuardian: GuardianClient, legacyInventory: {key: string, provenance: string}[], legacyPrepared?: {ownerState: JsonValue, replacementId: string}, legacySnapshot: OwnerRecoverySnapshot, prepared: {ownerState: JsonValue, replacementId: string}, recoverySnapshot: OwnerRecoverySnapshot}} LegacyOwnerBridge
|
|
42
38
|
*/
|
|
43
39
|
|
|
44
40
|
export default class RollbridgeDaemon {
|
|
@@ -150,7 +146,7 @@ export default class RollbridgeDaemon {
|
|
|
150
146
|
this.logger("release generation transition recovery failed", {error: error instanceof Error ? error.message : String(error), releaseId: transition.candidateReleaseId})
|
|
151
147
|
}
|
|
152
148
|
return {}
|
|
153
|
-
}
|
|
149
|
+
})
|
|
154
150
|
} finally {
|
|
155
151
|
this.generationTransitionRecovery = false
|
|
156
152
|
}
|
|
@@ -207,7 +203,7 @@ export default class RollbridgeDaemon {
|
|
|
207
203
|
if (!snapshot || snapshot.recovery?.guardian || !exactBootstrap) throw new Error("Retired owner reset requires an exact committed bootstrap with missing guardian identity")
|
|
208
204
|
const control = await inspectControlSocket(this.config.control.path)
|
|
209
205
|
if (control.alive) throw new Error(`Retired owner reset refuses responsive control owner at ${this.config.control.path}`)
|
|
210
|
-
this.guardianIdentity = {socketPath:
|
|
206
|
+
this.guardianIdentity = {socketPath: `${this.statePath}.guardian.sock`, token: crypto.randomBytes(32).toString("hex")}
|
|
211
207
|
this.guardian = new GuardianClient(this.guardianIdentity)
|
|
212
208
|
await this.guardian.launch()
|
|
213
209
|
this.guardianIdentity.pid = this.guardian.pid
|
|
@@ -225,12 +221,10 @@ export default class RollbridgeDaemon {
|
|
|
225
221
|
|
|
226
222
|
if (snapshot && !recovery) throw new Error(`Owner recovery state ${this.statePath} is missing durable guardian identity; refusing to overwrite it.`)
|
|
227
223
|
|
|
228
|
-
const guardianIdentity = recovery?.guardian
|
|
229
|
-
|
|
230
|
-
:
|
|
231
|
-
|
|
232
|
-
token: crypto.randomBytes(32).toString("hex")
|
|
233
|
-
}
|
|
224
|
+
const guardianIdentity = recovery?.guardian || {
|
|
225
|
+
socketPath: `${this.statePath}.guardian.sock`,
|
|
226
|
+
token: crypto.randomBytes(32).toString("hex")
|
|
227
|
+
}
|
|
234
228
|
this.guardianIdentity = guardianIdentity
|
|
235
229
|
this.guardian = new GuardianClient(guardianIdentity)
|
|
236
230
|
if (recovery) {
|
|
@@ -514,7 +508,7 @@ export default class RollbridgeDaemon {
|
|
|
514
508
|
const persisted = state && typeof state === "object" && !Array.isArray(state) ? /** @type {OwnerRecoverySnapshot} */ (state) : undefined
|
|
515
509
|
|
|
516
510
|
if (!persisted?.recovery?.guardian) throw new Error(`No committed durable owner transaction was found at ${this.statePath}`)
|
|
517
|
-
this.guardianIdentity =
|
|
511
|
+
this.guardianIdentity = persisted.recovery.guardian
|
|
518
512
|
this.guardian = new GuardianClient(this.guardianIdentity)
|
|
519
513
|
await this.guardian.connect()
|
|
520
514
|
this.watchOwnerReplacementEvents()
|
|
@@ -522,16 +516,9 @@ export default class RollbridgeDaemon {
|
|
|
522
516
|
configDigest: persisted.recovery.configDigest,
|
|
523
517
|
runtime: persisted.daemonRuntime ? {...persisted.daemonRuntime} : null
|
|
524
518
|
}
|
|
525
|
-
const
|
|
526
|
-
const versionDrift = Boolean(this.runtime?.version && guardianVersion && guardianVersion !== this.runtime.version)
|
|
527
|
-
const replacementProtocol = versionDrift ? "legacy" : await this.guardian.ownerReplacementProtocol()
|
|
519
|
+
const replacementProtocol = await this.guardian.ownerReplacementProtocol()
|
|
528
520
|
const legacyBridge = replacementProtocol === "legacy"
|
|
529
|
-
? await this.prepareLegacyOwnerReplacement({
|
|
530
|
-
guardianVersion,
|
|
531
|
-
persisted,
|
|
532
|
-
persistedAuthority,
|
|
533
|
-
protocol: versionDrift ? "guardian-version-drift" : "legacy-protocol"
|
|
534
|
-
})
|
|
521
|
+
? await this.prepareLegacyOwnerReplacement({persisted, persistedAuthority})
|
|
535
522
|
: undefined
|
|
536
523
|
const prepared = legacyBridge?.prepared ?? await this.guardian.prepareOwnerReplacement(persistedAuthority, this.ownerAuthority())
|
|
537
524
|
let preparedStatus
|
|
@@ -777,33 +764,6 @@ export default class RollbridgeDaemon {
|
|
|
777
764
|
}
|
|
778
765
|
}
|
|
779
766
|
|
|
780
|
-
/**
|
|
781
|
-
* Reads the retained guardian's package version without deciding the replacement protocol.
|
|
782
|
-
*
|
|
783
|
-
* The version is read from the guardian's own executable rather than its capabilities
|
|
784
|
-
* command: a pre-replacement guardian only answers that command once a replacement
|
|
785
|
-
* transaction is prepared, so asking here would move the probe ahead of the transaction
|
|
786
|
-
* every capability-gated guardian depends on. A guardian whose version cannot be read
|
|
787
|
-
* stays on the protocol classification path.
|
|
788
|
-
* @returns {Promise<string | undefined>} Retained guardian package version.
|
|
789
|
-
*/
|
|
790
|
-
async retainedGuardianVersion() {
|
|
791
|
-
if (!this.guardian) throw new Error("Guardian version reconciliation requires the durable process guardian")
|
|
792
|
-
return await guardianVersionFromProcess(this.guardian.pid)
|
|
793
|
-
}
|
|
794
|
-
|
|
795
|
-
/**
|
|
796
|
-
* Names the last-resort refusal for a guardian this daemon cannot hand off automatically.
|
|
797
|
-
* @param {string | undefined} guardianVersion - Retained guardian package version.
|
|
798
|
-
* @param {string} cause - Measured reason the automatic handoff cannot run.
|
|
799
|
-
* @returns {Error} Refusal naming both versions and the exact repair command.
|
|
800
|
-
*/
|
|
801
|
-
guardianVersionHandoffRefusal(guardianVersion, cause) {
|
|
802
|
-
const configArgument = this.configPath ? ` --config ${this.configPath}` : ""
|
|
803
|
-
|
|
804
|
-
return new Error(`The persistent Rollbridge guardian is version ${guardianVersion ?? "unknown"} while this daemon runs rollbridge ${this.runtime?.version ?? "unknown"}; ${cause}. Run "rollbridge shutdown${configArgument}" and then "rollbridge ensure-daemon${configArgument}" to replace the supervisor with this Rollbridge installation.`)
|
|
805
|
-
}
|
|
806
|
-
|
|
807
767
|
/** Requires the persistent guardian protocol which can restart an accepted daemon. */
|
|
808
768
|
async assertGuardianDaemonRecoveryCapability() {
|
|
809
769
|
if (!this.guardian) throw new Error("Owner recovery capability check requires the durable process guardian")
|
|
@@ -812,18 +772,10 @@ export default class RollbridgeDaemon {
|
|
|
812
772
|
try {
|
|
813
773
|
capabilities = await this.guardian.capabilities()
|
|
814
774
|
} catch (error) {
|
|
815
|
-
if (!(error instanceof Error) || !
|
|
775
|
+
if (!(error instanceof Error) || !["Guardian capabilities requires a process key", "Unknown guardian command: capabilities"].includes(error.message)) throw error
|
|
816
776
|
throw new Error("The persistent Rollbridge guardian predates daemon recovery. Perform one explicit clean Rollbridge shutdown and restart before retrying this package upgrade.", {cause: error})
|
|
817
777
|
}
|
|
818
778
|
if (capabilities.daemonRecovery !== 1) throw new Error("The persistent Rollbridge guardian does not support the required daemon recovery protocol")
|
|
819
|
-
const clientVersion = this.runtime?.version
|
|
820
|
-
const guardianVersion = capabilities.guardianVersion
|
|
821
|
-
|
|
822
|
-
if (!clientVersion) return
|
|
823
|
-
if (!guardianVersion) throw this.guardianVersionHandoffRefusal(guardianVersion, "its version cannot be attested, so no handoff can be authorised")
|
|
824
|
-
if (guardianVersion !== clientVersion) {
|
|
825
|
-
throw this.guardianVersionHandoffRefusal(guardianVersion, "this startup path has no incumbent daemon to hand guardian ownership from")
|
|
826
|
-
}
|
|
827
779
|
}
|
|
828
780
|
|
|
829
781
|
/**
|
|
@@ -944,15 +896,13 @@ export default class RollbridgeDaemon {
|
|
|
944
896
|
}
|
|
945
897
|
|
|
946
898
|
/**
|
|
947
|
-
* Authenticates and prepares the disruptive
|
|
948
|
-
*
|
|
949
|
-
* @param {{guardianVersion?: string, persisted: OwnerRecoverySnapshot, persistedAuthority: {configDigest: string, runtime: import("./daemon-runtime.js").DaemonRuntimeIdentity | null}, protocol: GuardianHandoffProtocol}} options - Bridge evidence and why the bridge is required.
|
|
899
|
+
* Authenticates and prepares the one-time disruptive bridge for an exact legacy guardian protocol.
|
|
900
|
+
* @param {{persisted: OwnerRecoverySnapshot, persistedAuthority: {configDigest: string, runtime: import("./daemon-runtime.js").DaemonRuntimeIdentity | null}}} options - Legacy evidence.
|
|
950
901
|
* @returns {Promise<LegacyOwnerBridge>} Prepared bridge.
|
|
951
902
|
*/
|
|
952
|
-
async prepareLegacyOwnerReplacement({
|
|
903
|
+
async prepareLegacyOwnerReplacement({persisted, persistedAuthority}) {
|
|
953
904
|
const legacyProcessKey = ownerSnapshotProcessKeys(persisted, persisted.singletonReleaseIds)[0]
|
|
954
905
|
const legacyGuardian = this.guardian
|
|
955
|
-
const versionHandoff = protocol === "guardian-version-drift"
|
|
956
906
|
let legacyPrepared
|
|
957
907
|
|
|
958
908
|
if (!legacyProcessKey) throw new Error("Legacy disruptive owner replacement requires an exact guardian-owned process registration in durable state")
|
|
@@ -985,23 +935,12 @@ export default class RollbridgeDaemon {
|
|
|
985
935
|
|
|
986
936
|
try {
|
|
987
937
|
if (persisted.recovery.configDigest !== this.ownerRecoveryConfigDigest()) {
|
|
988
|
-
if (versionHandoff) throw this.guardianVersionHandoffRefusal(guardianVersion, "the handoff keeps the incumbent config identity unchanged and this deploy changes config authority")
|
|
989
938
|
throw new Error("The one-time legacy guardian bridge requires the incumbent config identity unchanged; retry the config change after the bridge establishes split-3 authority")
|
|
990
939
|
}
|
|
991
|
-
if (!this.legacyIncumbentPid)
|
|
992
|
-
if (versionHandoff) throw this.guardianVersionHandoffRefusal(guardianVersion, "ensure-daemon reported no incumbent daemon to hand guardian ownership from")
|
|
993
|
-
throw new Error("The authenticated legacy guardian requires an exact incumbent PID from ensure-daemon for the disruptive bridge")
|
|
994
|
-
}
|
|
940
|
+
if (!this.legacyIncumbentPid) throw new Error("The authenticated legacy guardian requires an exact incumbent PID from ensure-daemon for the disruptive bridge")
|
|
995
941
|
if (!this.guardianIdentity?.pid) throw new Error("The authenticated legacy guardian state is missing its exact guardian PID")
|
|
996
942
|
await verifyLegacyGuardianProcess(this.guardianIdentity.pid, this.guardianIdentity.socketPath)
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
try {
|
|
1000
|
-
incumbentStartTime = await verifyLegacyDaemonProcess(this.legacyIncumbentPid, this.configPath, persisted.control.path)
|
|
1001
|
-
} catch (error) {
|
|
1002
|
-
if (!versionHandoff) throw error
|
|
1003
|
-
throw this.guardianVersionHandoffRefusal(guardianVersion, `the incumbent daemon PID ${this.legacyIncumbentPid} is not the attested owner: ${error instanceof Error ? error.message : String(error)}`)
|
|
1004
|
-
}
|
|
943
|
+
const incumbentStartTime = await verifyLegacyDaemonProcess(this.legacyIncumbentPid, this.configPath, persisted.control.path)
|
|
1005
944
|
if (legacyPrepared) assertLegacyPrivateOwnerState(legacyPrepared.ownerState, persisted, persistedAuthority)
|
|
1006
945
|
try {
|
|
1007
946
|
incumbentControl = await openControlSession(persisted.control.path)
|
|
@@ -1028,9 +967,7 @@ export default class RollbridgeDaemon {
|
|
|
1028
967
|
|
|
1029
968
|
assertLegacyGuardianInventoryMembership(legacyInventory, /** @type {PrivateOwnerState} */ (ownerState))
|
|
1030
969
|
const upgradedIdentity = /** @type {{pid?: number, socketPath: string, token: string}} */ ({
|
|
1031
|
-
socketPath:
|
|
1032
|
-
? `${this.statePath}.guardian-${this.runtime?.version}.sock`
|
|
1033
|
-
: `${this.statePath}.split3-guardian.sock`),
|
|
970
|
+
socketPath: `${this.statePath}.split3-guardian.sock`,
|
|
1034
971
|
token: crypto.randomBytes(32).toString("hex")
|
|
1035
972
|
})
|
|
1036
973
|
|
|
@@ -1045,15 +982,12 @@ export default class RollbridgeDaemon {
|
|
|
1045
982
|
recovery: {...persisted.recovery, guardian: upgradedIdentity}
|
|
1046
983
|
})
|
|
1047
984
|
|
|
1048
|
-
this.logger("
|
|
985
|
+
this.logger("legacy owner replacement bridge prepared", {
|
|
1049
986
|
guardianPid: upgradedIdentity.pid ?? null,
|
|
1050
|
-
guardianVersion: guardianVersion ?? "unknown",
|
|
1051
987
|
incumbentPid: this.legacyIncumbentPid,
|
|
1052
|
-
|
|
1053
|
-
replacementId: prepared.replacementId,
|
|
1054
|
-
runtimeVersion: this.runtime?.version ?? "unknown"
|
|
988
|
+
replacementId: prepared.replacementId
|
|
1055
989
|
})
|
|
1056
|
-
return {boundaryCrossed: false,
|
|
990
|
+
return {boundaryCrossed: false, incumbentControl, incumbentStartTime, legacyGuardian, legacyInventory, legacyPrepared, legacySnapshot: persisted, prepared, recoverySnapshot}
|
|
1057
991
|
} catch (upgradeError) {
|
|
1058
992
|
incumbentControl?.close()
|
|
1059
993
|
let cleanupError
|
|
@@ -1133,20 +1067,10 @@ export default class RollbridgeDaemon {
|
|
|
1133
1067
|
|
|
1134
1068
|
if (staged.committed) throw new Error("Legacy guardian committed replacement before the authenticated disruptive boundary")
|
|
1135
1069
|
}
|
|
1136
|
-
|
|
1137
|
-
? `retained guardian ${bridge.guardianVersion ?? "unknown"} predates this daemon runtime ${this.runtime?.version ?? "unknown"}`
|
|
1138
|
-
: "retained guardian and daemon lacked atomic replacement protocol"
|
|
1139
|
-
|
|
1140
|
-
this.logger("guardian upgrade bridge crossed", {
|
|
1070
|
+
this.logger("legacy owner replacement disruptive boundary", {
|
|
1141
1071
|
disruptive: true,
|
|
1142
|
-
guardianVersion: bridge.guardianVersion ?? "unknown",
|
|
1143
1072
|
incumbentPid: this.legacyIncumbentPid,
|
|
1144
|
-
|
|
1145
|
-
reason: boundaryReason,
|
|
1146
|
-
// The retained guardian keeps its own children; the boundary retires only the incumbent daemon.
|
|
1147
|
-
retainedGuardianPid: bridge.legacyGuardian.pid ?? null,
|
|
1148
|
-
retainedGuardianProcesses: bridge.legacyInventory.length,
|
|
1149
|
-
runtimeVersion: this.runtime?.version ?? "unknown"
|
|
1073
|
+
reason: "retained guardian and daemon lacked atomic replacement protocol"
|
|
1150
1074
|
})
|
|
1151
1075
|
await this.guardian.beginLegacyOwnerClaim(
|
|
1152
1076
|
bridge.prepared.replacementId,
|
|
@@ -1162,8 +1086,8 @@ export default class RollbridgeDaemon {
|
|
|
1162
1086
|
await this.guardian.completeLegacyOwnerClaim(bridge.prepared.replacementId)
|
|
1163
1087
|
this.ownerTransition = /** @type {OwnerTransition} */ ({
|
|
1164
1088
|
disruptive: true,
|
|
1165
|
-
mode:
|
|
1166
|
-
reason:
|
|
1089
|
+
mode: "legacy-first-upgrade",
|
|
1090
|
+
reason: "retained guardian and daemon lacked atomic replacement protocol"
|
|
1167
1091
|
})
|
|
1168
1092
|
}
|
|
1169
1093
|
|
|
@@ -1465,7 +1389,7 @@ export default class RollbridgeDaemon {
|
|
|
1465
1389
|
releaseId: stringOrUndefined(data.releaseId),
|
|
1466
1390
|
releasePath: requiredString(data.releasePath, "releasePath"),
|
|
1467
1391
|
revision: stringOrUndefined(data.revision)
|
|
1468
|
-
})
|
|
1392
|
+
}))
|
|
1469
1393
|
}
|
|
1470
1394
|
|
|
1471
1395
|
if (commandName === "recover-generation-transition") {
|
|
@@ -1475,22 +1399,11 @@ export default class RollbridgeDaemon {
|
|
|
1475
1399
|
releaseId: requiredString(data.releaseId, "releaseId"),
|
|
1476
1400
|
releasePath: requiredString(data.releasePath, "releasePath"),
|
|
1477
1401
|
revision: requiredString(data.revision, "revision")
|
|
1478
|
-
})
|
|
1402
|
+
}))
|
|
1479
1403
|
}
|
|
1480
1404
|
|
|
1481
1405
|
if (commandName === "status") {
|
|
1482
|
-
|
|
1483
|
-
let guardian
|
|
1484
|
-
|
|
1485
|
-
try {
|
|
1486
|
-
const diagnostics = await this.guardian?.guardianStatus()
|
|
1487
|
-
|
|
1488
|
-
guardian = diagnostics ? {guardianPid: diagnostics.guardianPid, ownerMutation: diagnostics.ownerMutation, version: diagnostics.version} : {guardianPid: null, ownerMutation: null, version: null}
|
|
1489
|
-
} catch (error) {
|
|
1490
|
-
guardian = {error: error instanceof Error ? error.message : String(error), guardianPid: this.guardianIdentity?.pid ?? null, ownerMutation: null, version: null}
|
|
1491
|
-
}
|
|
1492
|
-
|
|
1493
|
-
return {...status, guardian}
|
|
1406
|
+
return this.status({includeLogs: statusIncludeLogs(data.includeLogs)})
|
|
1494
1407
|
}
|
|
1495
1408
|
|
|
1496
1409
|
if (commandName === "events") {
|
|
@@ -1501,18 +1414,18 @@ export default class RollbridgeDaemon {
|
|
|
1501
1414
|
return await this.executeOwnerMutation("stop", async () => {
|
|
1502
1415
|
await this.stopRelease(stringOrUndefined(data.releaseId))
|
|
1503
1416
|
return this.status()
|
|
1504
|
-
}
|
|
1417
|
+
})
|
|
1505
1418
|
}
|
|
1506
1419
|
|
|
1507
1420
|
if (commandName === "restart") {
|
|
1508
1421
|
return await this.executeOwnerMutation("restart", async () => await this.restartProcesses({
|
|
1509
1422
|
policy: stringOrUndefined(data.policy),
|
|
1510
1423
|
processId: stringOrUndefined(data.processId)
|
|
1511
|
-
})
|
|
1424
|
+
}))
|
|
1512
1425
|
}
|
|
1513
1426
|
|
|
1514
1427
|
if (commandName === "rollback") {
|
|
1515
|
-
return await this.executeOwnerMutation("rollback", async () => await this.rollback({releaseId: stringOrUndefined(data.releaseId)})
|
|
1428
|
+
return await this.executeOwnerMutation("rollback", async () => await this.rollback({releaseId: stringOrUndefined(data.releaseId)}))
|
|
1516
1429
|
}
|
|
1517
1430
|
|
|
1518
1431
|
if (commandName === "shutdown") {
|
|
@@ -1576,13 +1489,11 @@ export default class RollbridgeDaemon {
|
|
|
1576
1489
|
* Runs one owner mutation under the guardian's prepare/commit exclusion fence.
|
|
1577
1490
|
* @param {string} operation - Mutation diagnostic name.
|
|
1578
1491
|
* @param {() => Promise<Record<string, JsonValue>>} callback - Mutating operation.
|
|
1579
|
-
* @param {{waitingFor?: string}} [options] - What this mutation blocks on, so a refused
|
|
1580
|
-
* later mutation names the real blocker instead of just "in progress".
|
|
1581
1492
|
* @returns {Promise<Record<string, JsonValue>>} Operation result.
|
|
1582
1493
|
*/
|
|
1583
|
-
async executeOwnerMutation(operation, callback
|
|
1494
|
+
async executeOwnerMutation(operation, callback) {
|
|
1584
1495
|
if (!this.guardian) return await callback()
|
|
1585
|
-
const mutationId = await this.guardian.beginOwnerMutation(operation
|
|
1496
|
+
const mutationId = await this.guardian.beginOwnerMutation(operation)
|
|
1586
1497
|
let result
|
|
1587
1498
|
let operationError
|
|
1588
1499
|
|
|
@@ -1718,46 +1629,6 @@ export default class RollbridgeDaemon {
|
|
|
1718
1629
|
await this.updateGenerationTransition("restoring_committed")
|
|
1719
1630
|
return await this.resumeGenerationTransition()
|
|
1720
1631
|
}
|
|
1721
|
-
await this.escalateWedgedGenerations(nextConfig, newReleaseId)
|
|
1722
|
-
// A release id that is still retained by a generation holding live work cannot be started
|
|
1723
|
-
// a second time: the candidate's processes re-register the guardian keys that the retained
|
|
1724
|
-
// generation's own processes still hold, so the deploy aborts partway through `start()`
|
|
1725
|
-
// with a bare `Guardian provenance mismatch` — after the caller has committed to this
|
|
1726
|
-
// revision and, for a restore, after the deployment pointer has already moved. Refusing
|
|
1727
|
-
// from the retained record before any process starts keeps the incumbent serving, names the
|
|
1728
|
-
// release the guardian reports as active, and leaves the retained generation intact for
|
|
1729
|
-
// `rollbridge status` to reconcile against. Journaled transition resume above is exempt:
|
|
1730
|
-
// it re-drives the transaction that already owns this release id.
|
|
1731
|
-
//
|
|
1732
|
-
// The escalation above runs first on purpose: it is what clears the residue that made this
|
|
1733
|
-
// guard refuse the measured 2026-11-15 and 2026-11-16 deploys. A wedge that escalation can
|
|
1734
|
-
// prove is kill-safe stops holding its keys and the handoff proceeds; a generation that
|
|
1735
|
-
// still holds live work is not killed and is named here instead of aborting mid-`start()`.
|
|
1736
|
-
const heldGeneration = this.retainedGenerationHoldingLiveWork(newReleaseId)
|
|
1737
|
-
|
|
1738
|
-
if (heldGeneration) {
|
|
1739
|
-
const activeReleaseId = this.activeRelease?.releaseId ?? null
|
|
1740
|
-
|
|
1741
|
-
this.logger("deploy refused: retained generation still holds live processes", {
|
|
1742
|
-
activeReleaseId,
|
|
1743
|
-
heldProcesses: heldGeneration.processes,
|
|
1744
|
-
releaseId: newReleaseId,
|
|
1745
|
-
retainedState: heldGeneration.state
|
|
1746
|
-
})
|
|
1747
|
-
|
|
1748
|
-
throw new Error(`Deploy refused before activation: release "${newReleaseId}" is retained in state ${heldGeneration.state} while ${heldGeneration.count} process${heldGeneration.count === 1 ? "" : "es"} of that generation still hold live work (${heldGeneration.hold.join("; ")}). Activating that release id again would re-register those processes under a new provenance and abort with a guardian provenance mismatch. The guardian's active release is ${activeReleaseId ? `"${activeReleaseId}"` : "none"}; deploy a new release id or reconcile the deployment pointer with \`rollbridge status\`.`)
|
|
1749
|
-
}
|
|
1750
|
-
const admission = this.evaluateGenerationAdmission(newReleaseId, nextConfig)
|
|
1751
|
-
|
|
1752
|
-
if (!admission.allowed) {
|
|
1753
|
-
this.logger("deploy refused by draining-generation bound", {
|
|
1754
|
-
candidateReleaseId: newReleaseId,
|
|
1755
|
-
draining: admission.draining.map((group) => ({releaseId: group.releaseId, status: group.status})),
|
|
1756
|
-
limit: nextConfig.releaseRetention.maxConcurrentDrainingReleaseGroups,
|
|
1757
|
-
wouldBeDrainingCount: admission.wouldBeDrainingCount
|
|
1758
|
-
})
|
|
1759
|
-
throw new Error(/** @type {string} */ (admission.reason))
|
|
1760
|
-
}
|
|
1761
1632
|
const release = new ReleaseGroup({
|
|
1762
1633
|
config: nextConfig,
|
|
1763
1634
|
logger: this.logger,
|
|
@@ -1814,79 +1685,6 @@ export default class RollbridgeDaemon {
|
|
|
1814
1685
|
return await this.resumeGenerationTransition()
|
|
1815
1686
|
}
|
|
1816
1687
|
|
|
1817
|
-
/**
|
|
1818
|
-
* Applies the draining-generation bound before a handoff.
|
|
1819
|
-
*
|
|
1820
|
-
* Without this, every deploy retires one more generation and the backlog grows
|
|
1821
|
-
* monotonically whenever a drain outlasts the deploy cadence; a consumer that checks the
|
|
1822
|
-
* count after activation reports the growth it cannot undo. Refusing here names the
|
|
1823
|
-
* draining groups and whether each is still progressing or wedged, before any process of
|
|
1824
|
-
* the candidate is started.
|
|
1825
|
-
* @param {string} candidateReleaseId - Release the handoff would activate.
|
|
1826
|
-
* @param {import("./config.js").RollbridgeConfig} config - Config governing the handoff.
|
|
1827
|
-
* @returns {import("./drain-policy.js").GenerationAdmission} Admission decision and the groups behind it.
|
|
1828
|
-
*/
|
|
1829
|
-
evaluateGenerationAdmission(candidateReleaseId, config) {
|
|
1830
|
-
const draining = [...this.releases.values()]
|
|
1831
|
-
.filter((release) => release.state === "draining")
|
|
1832
|
-
.map((release) => {
|
|
1833
|
-
const drain = release.drainStatus()
|
|
1834
|
-
|
|
1835
|
-
return {reason: drain?.reason ?? "no liveness observation recorded", releaseId: release.releaseId, status: drain?.status ?? "unproven"}
|
|
1836
|
-
})
|
|
1837
|
-
|
|
1838
|
-
return admitGenerationRelease({
|
|
1839
|
-
candidateReleaseId,
|
|
1840
|
-
draining,
|
|
1841
|
-
incumbentWillDrain: this.activeRelease !== undefined,
|
|
1842
|
-
maxConcurrentDrainingReleaseGroups: config.releaseRetention.maxConcurrentDrainingReleaseGroups
|
|
1843
|
-
})
|
|
1844
|
-
}
|
|
1845
|
-
|
|
1846
|
-
/**
|
|
1847
|
-
* Reports whether a retained release group still holds live work that a re-activation would collide with.
|
|
1848
|
-
*
|
|
1849
|
-
* `stopped` is not the same claim as `drained`: the guardian's provenance fence only lets a
|
|
1850
|
-
* process key be re-registered once the record that holds it is gone. A generation recorded
|
|
1851
|
-
* as drained while its process group is still alive — the shape an indefinite worker drain
|
|
1852
|
-
* produces — still holds its keys, so its release id is not available for a restore.
|
|
1853
|
-
* @param {string} releaseId - Release id a handoff would activate.
|
|
1854
|
-
* @returns {HeldGenerationHold | undefined} The live hold, or undefined when the id is free.
|
|
1855
|
-
*/
|
|
1856
|
-
retainedGenerationHoldingLiveWork(releaseId) {
|
|
1857
|
-
const retained = this.releases.get(releaseId)
|
|
1858
|
-
|
|
1859
|
-
if (!retained || retained === this.activeRelease) return undefined
|
|
1860
|
-
|
|
1861
|
-
return heldGenerationHold(retained)
|
|
1862
|
-
}
|
|
1863
|
-
|
|
1864
|
-
/**
|
|
1865
|
-
* Gives every retiring generation the escalation decision its own retirement left open,
|
|
1866
|
-
* before the handoff is judged.
|
|
1867
|
-
*
|
|
1868
|
-
* A deploy is the event that makes a wedged generation expensive: until something tries to
|
|
1869
|
-
* activate a successor, a retired release holding a live process group costs nothing
|
|
1870
|
-
* observable. Acting here means the observation that proves the wedge and the action on it
|
|
1871
|
-
* come from the same read the admission bound already performs, so no sampler has to run
|
|
1872
|
-
* alongside the daemon. Generations that already recorded `release drained` are included:
|
|
1873
|
-
* their retirement finished, and the process group they left behind is the residue that
|
|
1874
|
-
* blocked the measured 2026-11-15 deploys.
|
|
1875
|
-
* @param {import("./config.js").RollbridgeConfig} config - Config governing the handoff.
|
|
1876
|
-
* @param {string} candidateReleaseId - Release the handoff would activate.
|
|
1877
|
-
* @returns {Promise<void>} Resolves once every retiring generation has been decided.
|
|
1878
|
-
*/
|
|
1879
|
-
async escalateWedgedGenerations(config, candidateReleaseId) {
|
|
1880
|
-
if (!config.releaseRetention.escalateWedgedGenerations) return
|
|
1881
|
-
|
|
1882
|
-
for (const release of this.releases.values()) {
|
|
1883
|
-
if (release.releaseId === candidateReleaseId) continue
|
|
1884
|
-
if (release.state !== "draining" && release.state !== "stopped" && release.state !== "failed") continue
|
|
1885
|
-
|
|
1886
|
-
await release.escalateWedgedGeneration({reason: `deploy of ${candidateReleaseId}`})
|
|
1887
|
-
}
|
|
1888
|
-
}
|
|
1889
|
-
|
|
1890
1688
|
/**
|
|
1891
1689
|
* Continues one exact durable generation transition without an internal retry loop.
|
|
1892
1690
|
* @returns {Promise<Record<string, JsonValue>>} Deploy result after commit.
|
|
@@ -1908,8 +1706,11 @@ export default class RollbridgeDaemon {
|
|
|
1908
1706
|
let retirementFailure = activationLifecycle ? undefined : previousRelease?.retirementError
|
|
1909
1707
|
|
|
1910
1708
|
if (transition.phase === "candidate_ready") {
|
|
1911
|
-
|
|
1912
|
-
|
|
1709
|
+
// Candidate processes are already started and health-checked. Keep the incumbent
|
|
1710
|
+
// serving until the candidate has acknowledged activation and traffic has switched;
|
|
1711
|
+
// retiring it here can stop the only live web/jobs generation before its successor
|
|
1712
|
+
// carries traffic (STOE-2269).
|
|
1713
|
+
await this.updateGenerationTransition("activating_candidate")
|
|
1913
1714
|
}
|
|
1914
1715
|
|
|
1915
1716
|
if (transition.phase === "retiring_previous") {
|
|
@@ -1983,34 +1784,36 @@ export default class RollbridgeDaemon {
|
|
|
1983
1784
|
|
|
1984
1785
|
if (transition.phase === "committed_pending") {
|
|
1985
1786
|
await this.refreshServiceDefinitions(release)
|
|
1986
|
-
if (!activationLifecycle && previousRelease?.state === "active") {
|
|
1987
|
-
try {
|
|
1988
|
-
const retirementConfig = previousRelease.preserveConfigOnRetirement ? previousRelease.config : release.config
|
|
1989
|
-
|
|
1990
|
-
await previousRelease.beginRetirement(retirementConfig)
|
|
1991
|
-
} catch (error) {
|
|
1992
|
-
retirementFailure = previousRelease.retirementError ?? (error instanceof Error ? error.message : String(error))
|
|
1993
|
-
this.logger("release retirement quiescence failed", {error: retirementFailure, releaseId: previousRelease.releaseId})
|
|
1994
|
-
}
|
|
1995
|
-
}
|
|
1996
1787
|
await this.checkpointGenerationTransition()
|
|
1997
|
-
if (previousRelease) {
|
|
1998
|
-
const retirementConfig = previousRelease.config
|
|
1999
|
-
|
|
2000
|
-
void this.drainAndPrune(previousRelease, retirementConfig)
|
|
2001
|
-
}
|
|
2002
1788
|
try {
|
|
2003
1789
|
await this.replaceSingletons(release)
|
|
2004
1790
|
} catch (error) {
|
|
2005
1791
|
await this.failGenerationTransition(error instanceof Error ? error.message : String(error))
|
|
2006
1792
|
throw error
|
|
2007
1793
|
}
|
|
1794
|
+
if (previousRelease) {
|
|
1795
|
+
const retirementConfig = previousRelease.preserveConfigOnRetirement ? previousRelease.config : release.config
|
|
1796
|
+
|
|
1797
|
+
// Traffic is already on the candidate and singleton ownership has been
|
|
1798
|
+
// transferred. Retirement is deliberately detached so a slow old generation
|
|
1799
|
+
// cannot hold the committed deploy open or race guardian-owned replacement.
|
|
1800
|
+
void this.drainAndPrune(previousRelease, retirementConfig)
|
|
1801
|
+
}
|
|
2008
1802
|
transition.phase = "committed"
|
|
2009
1803
|
transition.error = undefined
|
|
2010
1804
|
transition.journalRevision = (transition.journalRevision ?? 0) + 1
|
|
2011
1805
|
transition.updatedAt = new Date().toISOString()
|
|
2012
1806
|
this.pruneStoppedReleases()
|
|
2013
1807
|
await this.checkpointGenerationTransition()
|
|
1808
|
+
if (previousRelease) {
|
|
1809
|
+
const retirementConfig = !activationLifecycle && previousRelease.preserveConfigOnRetirement ? previousRelease.config : release.config
|
|
1810
|
+
|
|
1811
|
+
// Traffic is already on the healthy candidate. Retirement is deliberately detached
|
|
1812
|
+
// from the owner mutation so a slow quiet/drain hook can never hold the deploy fence
|
|
1813
|
+
// or make the active successor unavailable. drainAndStop applies the bounded force
|
|
1814
|
+
// deadline and records any retirement failure on the old generation.
|
|
1815
|
+
void this.drainAndPrune(previousRelease, retirementConfig)
|
|
1816
|
+
}
|
|
2014
1817
|
}
|
|
2015
1818
|
|
|
2016
1819
|
return {
|
|
@@ -2040,8 +1843,6 @@ export default class RollbridgeDaemon {
|
|
|
2040
1843
|
if (!candidate) throw new Error(`Generation transition candidate ${transition.candidateReleaseId} is not retained`)
|
|
2041
1844
|
if (!previous) throw new Error(`Generation transition previous release ${transition.previousReleaseId} is not retained`)
|
|
2042
1845
|
if (this.activeRelease !== previous) throw new Error(`Previous release ${previous.releaseId} is not the authoritative proxy target`)
|
|
2043
|
-
/** @type {Promise<void> | undefined} */
|
|
2044
|
-
let candidateDrain
|
|
2045
1846
|
|
|
2046
1847
|
if (transition.phase === "activating_candidate") {
|
|
2047
1848
|
if (!transition.error && !transition.activationError) throw new Error("Candidate activation has no recorded failure to compensate")
|
|
@@ -2073,15 +1874,6 @@ export default class RollbridgeDaemon {
|
|
|
2073
1874
|
transition.compensationError = undefined
|
|
2074
1875
|
await this.updateGenerationTransition("restoring_previous")
|
|
2075
1876
|
this.logger("release generation compensation candidate retired", {releaseId: candidate.releaseId})
|
|
2076
|
-
|
|
2077
|
-
// The candidate's bounded drain starts here, before the restoration attempt. The only other
|
|
2078
|
-
// compensation drain runs after the success path, so when incumbent restoration throws the
|
|
2079
|
-
// candidate is left `draining` with a live process: measured 2026-11-17, candidate
|
|
2080
|
-
// `20261117063703` still held `web` pid 3201928 at 07:30Z after its compensation threw at
|
|
2081
|
-
// 06:42:31Z, and reached `release drained` only when a later compensation reached the
|
|
2082
|
-
// success path at 08:47:00Z. A retained generation holding live work is what refuses every
|
|
2083
|
-
// later deploy, and a failed candidate is never a restoration target.
|
|
2084
|
-
candidateDrain = this.drainAndPrune(candidate, candidate.config)
|
|
2085
1877
|
}
|
|
2086
1878
|
|
|
2087
1879
|
if (transition.phase === "restoring_previous") {
|
|
@@ -2093,12 +1885,10 @@ export default class RollbridgeDaemon {
|
|
|
2093
1885
|
|
|
2094
1886
|
transition.compensationError = failure
|
|
2095
1887
|
await this.failGenerationTransition(`Incumbent ${previous.releaseId} restoration failed: ${failure}`)
|
|
2096
|
-
await previous.stopProxyTraffic()
|
|
2097
1888
|
this.logger("release generation compensation restoration failed", {
|
|
2098
1889
|
activationError: transition.activationError,
|
|
2099
1890
|
error: failure,
|
|
2100
|
-
releaseId: previous.releaseId
|
|
2101
|
-
traffic: "unavailable"
|
|
1891
|
+
releaseId: previous.releaseId
|
|
2102
1892
|
})
|
|
2103
1893
|
throw new Error(`incumbent ${previous.releaseId} restoration failed: ${failure}`, {cause: error})
|
|
2104
1894
|
}
|
|
@@ -2137,7 +1927,7 @@ export default class RollbridgeDaemon {
|
|
|
2137
1927
|
this.logger("release generation compensation checkpoint failed", {error: failure, releaseId: candidate.releaseId})
|
|
2138
1928
|
throw new Error(`compensation checkpoint clear failed: ${failure}`, {cause: error})
|
|
2139
1929
|
}
|
|
2140
|
-
|
|
1930
|
+
void this.drainAndPrune(candidate, candidate.config)
|
|
2141
1931
|
this.logger("release generation compensation completed", result)
|
|
2142
1932
|
return result
|
|
2143
1933
|
}
|
|
@@ -2161,19 +1951,6 @@ export default class RollbridgeDaemon {
|
|
|
2161
1951
|
recoveryStatus: "already_recovered"
|
|
2162
1952
|
}
|
|
2163
1953
|
}
|
|
2164
|
-
// A committed journal describes the transition that already published the current
|
|
2165
|
-
// incumbent, not an in-flight transition to recover. When the recovery fence names
|
|
2166
|
-
// that live incumbent, the failed deploy never journaled its own transition, so the
|
|
2167
|
-
// only fence that describes the authoritative release is the completed one: recovery
|
|
2168
|
-
// is already done. In-flight phases keep the stale-fence guard failing closed.
|
|
2169
|
-
if (transition.phase === "committed" && transition.candidateReleaseId === identity.previousReleaseId && this.activeRelease?.releaseId === identity.previousReleaseId && this.releases.get(transition.candidateReleaseId) === this.activeRelease) {
|
|
2170
|
-
return {
|
|
2171
|
-
activeReleaseId: this.activeRelease.releaseId,
|
|
2172
|
-
candidateReleaseId: identity.releaseId,
|
|
2173
|
-
previousReleaseId: identity.previousReleaseId,
|
|
2174
|
-
recoveryStatus: "already_recovered"
|
|
2175
|
-
}
|
|
2176
|
-
}
|
|
2177
1954
|
if (transition.previousReleaseId !== identity.previousReleaseId) {
|
|
2178
1955
|
throw new Error(`Generation transition previous release is ${transition.previousReleaseId}; refusing stale recovery for ${identity.previousReleaseId}`)
|
|
2179
1956
|
}
|
|
@@ -2620,38 +2397,11 @@ export default class RollbridgeDaemon {
|
|
|
2620
2397
|
throw new Error(`Release "${target.releaseId}" is already active.`)
|
|
2621
2398
|
}
|
|
2622
2399
|
|
|
2623
|
-
// A generation recorded as `stopped` whose processes are still alive is the shape a
|
|
2624
|
-
// restore cannot honour: re-deploying that release id re-registers its guardian process
|
|
2625
|
-
// keys under a new provenance while the old processes still hold them, and the guardian
|
|
2626
|
-
// answers with a bare provenance mismatch. Use the guardian's own fence here so the
|
|
2627
|
-
// refusal is decided before anything is touched, and name the divergence — including a
|
|
2628
|
-
// process group that outlived the tracked process and is invisible as a pid — together
|
|
2629
|
-
// with the release the guardian actually reports as active, so the deployment tool can
|
|
2630
|
-
// reconcile its own current pointer instead of restoring a generation that already drained.
|
|
2631
|
-
// A target that is still draining is not refused here: stopping it is what makes its id
|
|
2632
|
-
// reusable, and the deploy below applies the same fence after that stop.
|
|
2633
|
-
const heldGeneration = target.state === "stopped" ? heldGenerationHold(target) : undefined
|
|
2634
|
-
|
|
2635
|
-
if (heldGeneration) {
|
|
2636
|
-
const activeReleaseId = this.activeRelease?.releaseId ?? null
|
|
2637
|
-
|
|
2638
|
-
this.logger("rollback refused: drained generation still holds live processes", {
|
|
2639
|
-
activeReleaseId,
|
|
2640
|
-
heldProcesses: heldGeneration.processes,
|
|
2641
|
-
releaseId: target.releaseId,
|
|
2642
|
-
revision: target.revision
|
|
2643
|
-
})
|
|
2644
|
-
|
|
2645
|
-
throw new Error(`Release "${target.releaseId}" is recorded as drained while ${heldGeneration.count} process${heldGeneration.count === 1 ? "" : "es"} of that generation still hold live work (${heldGeneration.hold.join("; ")}). Restoring it would re-register those processes under a new provenance and fail with a guardian provenance mismatch. The guardian's active release is ${activeReleaseId ? `"${activeReleaseId}"` : "none"}; reconcile the deployment pointer with \`rollbridge status\` before restoring.`)
|
|
2646
|
-
}
|
|
2647
|
-
|
|
2648
2400
|
// The target may still be draining a prior deploy (live processes). Stop it before the
|
|
2649
2401
|
// deploy below re-uses its id in this.releases, otherwise the still-running instance
|
|
2650
2402
|
// would be dropped from status/pruning/shutdown and could be orphaned.
|
|
2651
|
-
// Rollback is an owner mutation too, so this stop carries the same retirement bound as
|
|
2652
|
-
// `stop`: a process the supervisor must not kill must not be able to hold the fence.
|
|
2653
2403
|
if (target.state !== "stopped" && target.state !== "failed") {
|
|
2654
|
-
await target.stop(
|
|
2404
|
+
await target.stop()
|
|
2655
2405
|
}
|
|
2656
2406
|
|
|
2657
2407
|
this.logger("rollback starting", {releaseId: target.releaseId, releasePath: target.releasePath})
|
|
@@ -2872,7 +2622,7 @@ export default class RollbridgeDaemon {
|
|
|
2872
2622
|
|
|
2873
2623
|
/**
|
|
2874
2624
|
* @param {string | undefined} releaseId - Release id, or active release when omitted.
|
|
2875
|
-
* @returns {Promise<void>} Resolves when stopped
|
|
2625
|
+
* @returns {Promise<void>} Resolves when stopped.
|
|
2876
2626
|
*/
|
|
2877
2627
|
async stopRelease(releaseId) {
|
|
2878
2628
|
this.assertNoUnresolvedGenerationTransition("stop")
|
|
@@ -2881,10 +2631,7 @@ export default class RollbridgeDaemon {
|
|
|
2881
2631
|
if (!release) throw new Error(`Release not found: ${releaseId || "active"}`)
|
|
2882
2632
|
if (release === this.activeRelease) this.activeRelease = undefined
|
|
2883
2633
|
|
|
2884
|
-
|
|
2885
|
-
// supervisor must not kill holds the fence for the whole drain and every later deploy is
|
|
2886
|
-
// refused. The retirement bound is the same one the deploy path already uses.
|
|
2887
|
-
await release.stop({retirementTimeoutMs: this.config.proxy.nonBlockingDrainTimeoutMs})
|
|
2634
|
+
await release.stop()
|
|
2888
2635
|
this.logger("release stopped", {releaseId: release.releaseId})
|
|
2889
2636
|
this.pruneStoppedReleases()
|
|
2890
2637
|
this.persistState()
|
|
@@ -3129,13 +2876,13 @@ export default class RollbridgeDaemon {
|
|
|
3129
2876
|
await captureShutdownError(cleanupErrors, "proxy close", async () => this.proxy.close())
|
|
3130
2877
|
const dependentStopResults = await Promise.allSettled([
|
|
3131
2878
|
...[...this.singletons.values()].map((processInstance) => processInstance.stop()),
|
|
3132
|
-
...[...this.startingReleases].map((release) => release.stop(
|
|
3133
|
-
...[...this.releases.values()].map((release) => release.stop(
|
|
2879
|
+
...[...this.startingReleases].map((release) => release.stop()),
|
|
2880
|
+
...[...this.releases.values()].map((release) => release.stop())
|
|
3134
2881
|
])
|
|
3135
2882
|
const serviceStopResults = await Promise.allSettled([...this.services.values()].map((processInstance) => processInstance.stop()))
|
|
3136
2883
|
const stopResults = [...dependentStopResults, ...serviceStopResults]
|
|
3137
2884
|
const guardian = this.guardian
|
|
3138
|
-
if (guardian) await captureShutdownError(cleanupErrors, "process guardian shutdown", () => guardian.shutdown(
|
|
2885
|
+
if (guardian) await captureShutdownError(cleanupErrors, "process guardian shutdown", () => guardian.shutdown())
|
|
3139
2886
|
await captureShutdownError(cleanupErrors, "proxy server close", () => this.closeServer(this.proxyServer))
|
|
3140
2887
|
|
|
3141
2888
|
// Wait for any in-flight write first so it can't recreate or overwrite the final state (no
|
|
@@ -3316,42 +3063,6 @@ function statusIncludeLogs(value) {
|
|
|
3316
3063
|
return value
|
|
3317
3064
|
}
|
|
3318
3065
|
|
|
3319
|
-
/**
|
|
3320
|
-
* Names the live work a retained release group still holds, using the guardian's own fence.
|
|
3321
|
-
*
|
|
3322
|
-
* A process counts as holding work when the guardian would refuse to hand its key to a new
|
|
3323
|
-
* provenance: a running or stopping process, or a record already `stopped` whose process group
|
|
3324
|
-
* outlived it and is invisible as a pid.
|
|
3325
|
-
* @param {ReleaseGroup} release - Retained generation to inspect.
|
|
3326
|
-
* @returns {HeldGenerationHold | undefined} The hold and its log payload, or undefined when every process is gone.
|
|
3327
|
-
*/
|
|
3328
|
-
function heldGenerationHold(release) {
|
|
3329
|
-
const heldProcesses = release.status().processes.filter((processStatus) => !childRecordIsGone(processStatus))
|
|
3330
|
-
|
|
3331
|
-
if (heldProcesses.length === 0) return undefined
|
|
3332
|
-
|
|
3333
|
-
return {
|
|
3334
|
-
count: heldProcesses.length,
|
|
3335
|
-
hold: heldProcesses.map((processStatus) => {
|
|
3336
|
-
if (processStatus.heldProcessGroup) {
|
|
3337
|
-
const {liveMemberCount, pgid} = processStatus.heldProcessGroup
|
|
3338
|
-
const members = liveMemberCount === undefined ? "" : ` with ${liveMemberCount} live member(s)`
|
|
3339
|
-
|
|
3340
|
-
return `${processStatus.id}: process group ${pgid}${members} still alive after the tracked process exited`
|
|
3341
|
-
}
|
|
3342
|
-
|
|
3343
|
-
return `${processStatus.id}: state ${processStatus.state}, pid ${processStatus.pid ?? "unknown"}`
|
|
3344
|
-
}),
|
|
3345
|
-
processes: heldProcesses.map((processStatus) => ({
|
|
3346
|
-
heldProcessGroupPgid: processStatus.heldProcessGroup?.pgid ?? null,
|
|
3347
|
-
pid: processStatus.pid ?? null,
|
|
3348
|
-
processId: processStatus.id,
|
|
3349
|
-
state: processStatus.state
|
|
3350
|
-
})),
|
|
3351
|
-
state: release.state
|
|
3352
|
-
}
|
|
3353
|
-
}
|
|
3354
|
-
|
|
3355
3066
|
/**
|
|
3356
3067
|
* @param {PrivateOwnerState} transfer - Authenticated private incumbent state.
|
|
3357
3068
|
* @param {GenerationTransition} transition - Unresolved generation transition.
|
|
@@ -3412,15 +3123,6 @@ export function isLegacyGuardianPrepareDiagnostic(diagnostic) {
|
|
|
3412
3123
|
diagnostic === "Unknown guardian command: prepare-owner-replacement"
|
|
3413
3124
|
}
|
|
3414
3125
|
|
|
3415
|
-
/**
|
|
3416
|
-
* Reports the capability diagnostics that identify a guardian predating daemon recovery.
|
|
3417
|
-
* @param {string} diagnostic - Guardian error message.
|
|
3418
|
-
* @returns {boolean} Whether the guardian predates the daemon recovery capability.
|
|
3419
|
-
*/
|
|
3420
|
-
function isGuardianDaemonRecoveryDiagnostic(diagnostic) {
|
|
3421
|
-
return diagnostic === "Guardian capabilities requires a process key" || diagnostic === "Unknown guardian command: capabilities"
|
|
3422
|
-
}
|
|
3423
|
-
|
|
3424
3126
|
/**
|
|
3425
3127
|
* @param {DaemonStatus} snapshot - Serialized owner process snapshot.
|
|
3426
3128
|
* @param {Record<string, string>} [singletonReleaseIds] - Exact singleton owner releases.
|
|
@@ -3743,7 +3445,7 @@ function requiredNonNegativeInteger(value, key) {
|
|
|
3743
3445
|
/**
|
|
3744
3446
|
* Selects stopped releases to prune by the retention policy, keeping the most recent.
|
|
3745
3447
|
* @param {PrunableRelease[]} releases - Status of all tracked releases, in deploy order (oldest first).
|
|
3746
|
-
* @param {
|
|
3448
|
+
* @param {import("./config.js").ReleaseRetentionConfig} policy - Retention policy.
|
|
3747
3449
|
* @param {number} now - Current epoch milliseconds.
|
|
3748
3450
|
* @returns {string[]} Release ids to remove.
|
|
3749
3451
|
*/
|
|
@@ -3768,7 +3470,7 @@ export function releasesToPrune(releases, policy, now) {
|
|
|
3768
3470
|
}
|
|
3769
3471
|
|
|
3770
3472
|
/**
|
|
3771
|
-
* @typedef {{alive: boolean, application?: string, activeReleaseId?: string | null,
|
|
3473
|
+
* @typedef {{alive: boolean, application?: string, activeReleaseId?: string | null, proxy?: {host: string, port: number}}} ControlSocketInspection
|
|
3772
3474
|
*/
|
|
3773
3475
|
|
|
3774
3476
|
/**
|
|
@@ -3843,7 +3545,7 @@ export async function inspectControlSocket(socketPath, timeoutMs = 1000) {
|
|
|
3843
3545
|
/**
|
|
3844
3546
|
* Parses a control status response line into a Rollbridge identity, if it is one.
|
|
3845
3547
|
* @param {string} line - JSON response line.
|
|
3846
|
-
* @returns {{application: string, activeReleaseId: string | null,
|
|
3548
|
+
* @returns {{application: string, activeReleaseId: string | null, proxy: {host: string, port: number} | undefined} | undefined} Identity, or undefined when unrecognized.
|
|
3847
3549
|
*/
|
|
3848
3550
|
function parseControlStatus(line) {
|
|
3849
3551
|
/** @type {JsonValue} */
|
|
@@ -3860,8 +3562,5 @@ function parseControlStatus(line) {
|
|
|
3860
3562
|
|
|
3861
3563
|
const proxy = "proxy" in parsed && parsed.proxy && typeof parsed.proxy === "object" && !Array.isArray(parsed.proxy) && typeof parsed.proxy.host === "string" && typeof parsed.proxy.port === "number" ? {host: parsed.proxy.host, port: parsed.proxy.port} : undefined
|
|
3862
3564
|
|
|
3863
|
-
|
|
3864
|
-
const guardian = guardianValue && typeof guardianValue.version === "string" || guardianValue && guardianValue.version === null ? {ownerMutation: guardianValue.ownerMutation && typeof guardianValue.ownerMutation === "object" && !Array.isArray(guardianValue.ownerMutation) ? guardianValue.ownerMutation : null, version: guardianValue.version} : undefined
|
|
3865
|
-
|
|
3866
|
-
return {activeReleaseId: typeof parsed.activeReleaseId === "string" ? parsed.activeReleaseId : null, application: parsed.application, guardian, proxy}
|
|
3565
|
+
return {activeReleaseId: typeof parsed.activeReleaseId === "string" ? parsed.activeReleaseId : null, application: parsed.application, proxy}
|
|
3867
3566
|
}
|