rollbridge 0.1.70 → 0.1.71
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -63
- package/TODO.md +0 -13
- package/docs/cli.md +3 -128
- package/docs/config.md +6 -177
- package/docs/troubleshooting.md +6 -97
- package/docs/velocious.md +1 -1
- package/docs/workers.md +0 -11
- package/examples/tensorbuzz.com.js +2 -3
- package/package.json +1 -1
- package/src/cli.js +2 -68
- package/src/config.js +8 -347
- package/src/daemon.js +65 -387
- package/src/doctor.js +0 -74
- package/src/guardian-client.js +8 -67
- package/src/health.js +8 -99
- package/src/managed-process.js +16 -197
- package/src/process-guardian.js +22 -212
- package/src/process-memory.js +2 -62
- package/src/release-group.js +37 -435
- package/tensorbuzz.yml +1 -20
- package/test/completion.test.js +2 -5
- package/test/config-validation.test.js +5 -135
- package/test/guardian-client.test.js +5 -119
- package/test/health.test.js +1 -74
- package/test/managed-process.test.js +1 -260
- package/test/owner-recovery.test.js +21 -125
- package/test/owner-replacement.test.js +14 -175
- package/test/process-memory.test.js +1 -47
- package/test/release-group.test.js +2 -408
- package/test/rollbridge.test.js +72 -737
- package/test/shutdown-completion.test.js +0 -52
- package/test/support/process.js +1 -3
- package/changelog.d/20261025-guardian-re-register-dead-generation.md +0 -13
- package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +0 -1
- package/changelog.d/20261104-compose-service-units.md +0 -21
- package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +0 -20
- package/changelog.d/20261106-release-dependency-boot-budgets.md +0 -32
- package/changelog.d/20261109-path-filtered-builds.md +0 -11
- package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +0 -26
- package/changelog.d/20261111-contention-sensitive-daemon-specs.md +0 -27
- package/changelog.d/20261113-bounded-nonblocking-retirement.md +0 -5
- package/changelog.d/20261115-bound-draining-release-groups.md +0 -62
- package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +0 -27
- package/changelog.d/20261115-wedged-generation-escalation.md +0 -44
- package/changelog.d/20261116-bounded-explicit-stop-retirement.md +0 -30
- package/changelog.d/20261116-config-json-canonical-shape.md +0 -11
- package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +0 -29
- package/changelog.d/20261116-guardian-version-handoff.md +0 -37
- package/changelog.d/20261116-reclaim-dead-owner-mutation.md +0 -7
- package/changelog.d/20261117-activate-before-incumbent-retirement.md +0 -1
- package/changelog.d/20261117-activation-path-drain-deadline.md +0 -35
- package/changelog.d/20261117-stop-companion-after-retired-dependency.md +0 -7
- package/docs/compose-services.md +0 -222
- package/src/compose-unit.js +0 -725
- package/src/drain-policy.js +0 -252
- package/src/process-order.js +0 -26
- package/test/compose-config.test.js +0 -160
- package/test/compose-docker.test.js +0 -297
- package/test/compose-unit.test.js +0 -415
- package/test/drain-policy.test.js +0 -328
- package/test/fixtures/deferred-boot-app.js +0 -106
- package/test/fixtures/gated-app.js +0 -50
- package/test/tensorbuzz-config.test.js +0 -228
package/src/daemon.js
CHANGED
|
@@ -9,10 +9,8 @@ import {isDeepStrictEqual} from "node:util"
|
|
|
9
9
|
import httpProxy from "http-proxy"
|
|
10
10
|
import {loadConfig} from "./config.js"
|
|
11
11
|
import {openControlSession} from "./control-client.js"
|
|
12
|
-
import {admitGenerationRelease} from "./drain-policy.js"
|
|
13
12
|
import EventLog from "./event-log.js"
|
|
14
|
-
import GuardianClient
|
|
15
|
-
import {childRecordIsGone} from "./managed-process.js"
|
|
13
|
+
import GuardianClient from "./guardian-client.js"
|
|
16
14
|
import ReleaseGroup from "./release-group.js"
|
|
17
15
|
import {clearState, isProcessAlive, liveProcesses, readState, writeState} from "./state-store.js"
|
|
18
16
|
import {resolveGroupId, resolveUserId} from "./system-ids.js"
|
|
@@ -25,20 +23,18 @@ const STATE_PERSIST_INTERVAL_MS = 5000
|
|
|
25
23
|
* @typedef {{releaseId?: string, releasePath: string, revision?: string}} DeployArgs
|
|
26
24
|
* @typedef {{attestation?: string, releaseId: string, releasePath: string, revision: string}} BootstrapIdentity
|
|
27
25
|
* @typedef {{id: string, process: import("./managed-process.js").ManagedProcessStatus}} ProcessStatus
|
|
28
|
-
* @typedef {{disruptive: true, mode: "
|
|
29
|
-
* @typedef {"guardian-version-drift" | "legacy-protocol"} GuardianHandoffProtocol
|
|
26
|
+
* @typedef {{disruptive: true, mode: "legacy-first-upgrade", reason: string}} OwnerTransition
|
|
30
27
|
* @typedef {"candidate_ready" | "retiring_previous" | "previous_retired" | "activating_candidate" | "restoring_previous" | "retiring_failed_candidate" | "degraded_active" | "committed_pending" | "committed" | "restoring_committed"} GenerationTransitionPhase
|
|
31
28
|
* @typedef {{activationError?: string, activationLifecycle?: boolean, candidateReleaseId: string, candidateReleasePath: string, candidateRevision: string, compensationError?: string, configDigest: string, degradedIncumbent?: boolean, error?: string, journalRevision?: number, phase: GenerationTransitionPhase, previousReleaseId: string | null, startedAt: string, updatedAt: string}} GenerationTransition
|
|
32
|
-
* @typedef {{activeReleaseId: string | null, application: string, bootstrap: BootstrapIdentity | undefined, control: import("./config.js").ControlConfig, daemonPid: number, daemonRuntime: import("./daemon-runtime.js").DaemonRuntimeIdentity | undefined, generationTransition?: GenerationTransition,
|
|
29
|
+
* @typedef {{activeReleaseId: string | null, application: string, bootstrap: BootstrapIdentity | undefined, control: import("./config.js").ControlConfig, daemonPid: number, daemonRuntime: import("./daemon-runtime.js").DaemonRuntimeIdentity | undefined, generationTransition?: GenerationTransition, ownerRecovery: {configDigest: string, ready: boolean} | undefined, ownerTransition?: OwnerTransition, orphans: {id: string, pid: number, releaseId: string | null}[], proxy: {host: string, port: number | undefined, upstreamHost: string}, releaseReferences: {releaseId: string, releasePath: string}[], releases: import("./release-group.js").ReleaseStatus[], services: ProcessStatus[], singletons: ProcessStatus[]}} DaemonStatus
|
|
33
30
|
* @typedef {Omit<import("./managed-process.js").ManagedProcessStatus, "logs">} ManagedProcessStatusWithoutLogs
|
|
34
31
|
* @typedef {Omit<import("./release-group.js").ReleaseStatus, "processes"> & {processes: ManagedProcessStatusWithoutLogs[]}} ReleaseStatusWithoutLogs
|
|
35
32
|
* @typedef {Omit<ProcessStatus, "process"> & {process: ManagedProcessStatusWithoutLogs}} ProcessStatusWithoutLogs
|
|
36
33
|
* @typedef {Omit<DaemonStatus, "releases" | "services" | "singletons"> & {releases: ReleaseStatusWithoutLogs[], services: ProcessStatusWithoutLogs[], singletons: ProcessStatusWithoutLogs[]}} DaemonStatusWithoutLogs
|
|
37
|
-
* @typedef {{count: number, hold: string[], processes: {heldProcessGroupPgid: number | null, pid: number | null, processId: string, state: import("./managed-process.js").ManagedProcessState}[], state: import("./release-group.js").ReleaseState}} HeldGenerationHold
|
|
38
34
|
* @typedef {{configDigest: string, format: number, guardian: {pid?: number, socketPath: string, token: string}, reconnectGraceMs: number}} OwnerRecoveryMetadata
|
|
39
35
|
* @typedef {DaemonStatus & {recovery: OwnerRecoveryMetadata, serviceReleaseIds?: Record<string, string>, singletonReleaseIds?: Record<string, string>}} OwnerRecoverySnapshot
|
|
40
36
|
* @typedef {{authority: JsonValue, config: import("./config.js").RollbridgeConfig, listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>, listenerSourceId?: string, recovery?: {command: JsonValue, reconnectGraceMs: number, startupTimeoutMs: number}, releaseConfigs?: Record<string, import("./config.js").RollbridgeConfig>, retiredListenerHandoff?: number, serviceReleaseIds?: Record<string, string>, singletonReleaseIds?: Record<string, string>, snapshot: OwnerRecoverySnapshot}} PrivateOwnerState
|
|
41
|
-
* @typedef {{boundaryCrossed: boolean,
|
|
37
|
+
* @typedef {{boundaryCrossed: boolean, incumbentControl?: Awaited<ReturnType<typeof openControlSession>>, incumbentStartTime: string, legacyGuardian: GuardianClient, legacyInventory: {key: string, provenance: string}[], legacyPrepared?: {ownerState: JsonValue, replacementId: string}, legacySnapshot: OwnerRecoverySnapshot, prepared: {ownerState: JsonValue, replacementId: string}, recoverySnapshot: OwnerRecoverySnapshot}} LegacyOwnerBridge
|
|
42
38
|
*/
|
|
43
39
|
|
|
44
40
|
export default class RollbridgeDaemon {
|
|
@@ -150,7 +146,7 @@ export default class RollbridgeDaemon {
|
|
|
150
146
|
this.logger("release generation transition recovery failed", {error: error instanceof Error ? error.message : String(error), releaseId: transition.candidateReleaseId})
|
|
151
147
|
}
|
|
152
148
|
return {}
|
|
153
|
-
}
|
|
149
|
+
})
|
|
154
150
|
} finally {
|
|
155
151
|
this.generationTransitionRecovery = false
|
|
156
152
|
}
|
|
@@ -207,7 +203,7 @@ export default class RollbridgeDaemon {
|
|
|
207
203
|
if (!snapshot || snapshot.recovery?.guardian || !exactBootstrap) throw new Error("Retired owner reset requires an exact committed bootstrap with missing guardian identity")
|
|
208
204
|
const control = await inspectControlSocket(this.config.control.path)
|
|
209
205
|
if (control.alive) throw new Error(`Retired owner reset refuses responsive control owner at ${this.config.control.path}`)
|
|
210
|
-
this.guardianIdentity = {socketPath:
|
|
206
|
+
this.guardianIdentity = {socketPath: `${this.statePath}.guardian.sock`, token: crypto.randomBytes(32).toString("hex")}
|
|
211
207
|
this.guardian = new GuardianClient(this.guardianIdentity)
|
|
212
208
|
await this.guardian.launch()
|
|
213
209
|
this.guardianIdentity.pid = this.guardian.pid
|
|
@@ -225,12 +221,10 @@ export default class RollbridgeDaemon {
|
|
|
225
221
|
|
|
226
222
|
if (snapshot && !recovery) throw new Error(`Owner recovery state ${this.statePath} is missing durable guardian identity; refusing to overwrite it.`)
|
|
227
223
|
|
|
228
|
-
const guardianIdentity = recovery?.guardian
|
|
229
|
-
|
|
230
|
-
:
|
|
231
|
-
|
|
232
|
-
token: crypto.randomBytes(32).toString("hex")
|
|
233
|
-
}
|
|
224
|
+
const guardianIdentity = recovery?.guardian || {
|
|
225
|
+
socketPath: `${this.statePath}.guardian.sock`,
|
|
226
|
+
token: crypto.randomBytes(32).toString("hex")
|
|
227
|
+
}
|
|
234
228
|
this.guardianIdentity = guardianIdentity
|
|
235
229
|
this.guardian = new GuardianClient(guardianIdentity)
|
|
236
230
|
if (recovery) {
|
|
@@ -506,36 +500,15 @@ export default class RollbridgeDaemon {
|
|
|
506
500
|
/**
|
|
507
501
|
* Prepares from private guardian state, starts listeners, then asks the committed owner
|
|
508
502
|
* to atomically fence itself and transfer authority.
|
|
509
|
-
* @returns {Promise<void>} Resolves after the replacement is committed.
|
|
510
503
|
*/
|
|
511
504
|
async replaceIncompatibleOwner() {
|
|
512
|
-
const maxAttempts = 3
|
|
513
|
-
let lastError
|
|
514
|
-
|
|
515
|
-
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
|
516
|
-
try {
|
|
517
|
-
return await this.replaceIncompatibleOwnerAttempt()
|
|
518
|
-
} catch (error) {
|
|
519
|
-
lastError = error
|
|
520
|
-
const diagnostic = error instanceof Error ? error.message : String(error)
|
|
521
|
-
|
|
522
|
-
if (!/^Guardian owner state changed after prepare \(prepared revision \d+, current revision \d+\); abort the stale candidate before preparing again$/.test(diagnostic) || attempt === maxAttempts) {
|
|
523
|
-
throw error
|
|
524
|
-
}
|
|
525
|
-
this.logger("owner replacement candidate became stale; re-preparing", {attempt, maxAttempts, error: diagnostic})
|
|
526
|
-
}
|
|
527
|
-
}
|
|
528
|
-
throw lastError
|
|
529
|
-
}
|
|
530
|
-
|
|
531
|
-
async replaceIncompatibleOwnerAttempt() {
|
|
532
505
|
if (!this.statePath || !this.config.ownerRecovery) throw new Error("Atomic owner replacement requires ownerRecovery and the same statePath transaction anchor")
|
|
533
506
|
this.controlCommandsReady = false
|
|
534
507
|
const state = await readState(this.statePath)
|
|
535
508
|
const persisted = state && typeof state === "object" && !Array.isArray(state) ? /** @type {OwnerRecoverySnapshot} */ (state) : undefined
|
|
536
509
|
|
|
537
510
|
if (!persisted?.recovery?.guardian) throw new Error(`No committed durable owner transaction was found at ${this.statePath}`)
|
|
538
|
-
this.guardianIdentity =
|
|
511
|
+
this.guardianIdentity = persisted.recovery.guardian
|
|
539
512
|
this.guardian = new GuardianClient(this.guardianIdentity)
|
|
540
513
|
await this.guardian.connect()
|
|
541
514
|
this.watchOwnerReplacementEvents()
|
|
@@ -543,16 +516,9 @@ export default class RollbridgeDaemon {
|
|
|
543
516
|
configDigest: persisted.recovery.configDigest,
|
|
544
517
|
runtime: persisted.daemonRuntime ? {...persisted.daemonRuntime} : null
|
|
545
518
|
}
|
|
546
|
-
const
|
|
547
|
-
const versionDrift = Boolean(this.runtime?.version && guardianVersion && guardianVersion !== this.runtime.version)
|
|
548
|
-
const replacementProtocol = versionDrift ? "legacy" : await this.guardian.ownerReplacementProtocol()
|
|
519
|
+
const replacementProtocol = await this.guardian.ownerReplacementProtocol()
|
|
549
520
|
const legacyBridge = replacementProtocol === "legacy"
|
|
550
|
-
? await this.prepareLegacyOwnerReplacement({
|
|
551
|
-
guardianVersion,
|
|
552
|
-
persisted,
|
|
553
|
-
persistedAuthority,
|
|
554
|
-
protocol: versionDrift ? "guardian-version-drift" : "legacy-protocol"
|
|
555
|
-
})
|
|
521
|
+
? await this.prepareLegacyOwnerReplacement({persisted, persistedAuthority})
|
|
556
522
|
: undefined
|
|
557
523
|
const prepared = legacyBridge?.prepared ?? await this.guardian.prepareOwnerReplacement(persistedAuthority, this.ownerAuthority())
|
|
558
524
|
let preparedStatus
|
|
@@ -798,33 +764,6 @@ export default class RollbridgeDaemon {
|
|
|
798
764
|
}
|
|
799
765
|
}
|
|
800
766
|
|
|
801
|
-
/**
|
|
802
|
-
* Reads the retained guardian's package version without deciding the replacement protocol.
|
|
803
|
-
*
|
|
804
|
-
* The version is read from the guardian's own executable rather than its capabilities
|
|
805
|
-
* command: a pre-replacement guardian only answers that command once a replacement
|
|
806
|
-
* transaction is prepared, so asking here would move the probe ahead of the transaction
|
|
807
|
-
* every capability-gated guardian depends on. A guardian whose version cannot be read
|
|
808
|
-
* stays on the protocol classification path.
|
|
809
|
-
* @returns {Promise<string | undefined>} Retained guardian package version.
|
|
810
|
-
*/
|
|
811
|
-
async retainedGuardianVersion() {
|
|
812
|
-
if (!this.guardian) throw new Error("Guardian version reconciliation requires the durable process guardian")
|
|
813
|
-
return await guardianVersionFromProcess(this.guardian.pid)
|
|
814
|
-
}
|
|
815
|
-
|
|
816
|
-
/**
|
|
817
|
-
* Names the last-resort refusal for a guardian this daemon cannot hand off automatically.
|
|
818
|
-
* @param {string | undefined} guardianVersion - Retained guardian package version.
|
|
819
|
-
* @param {string} cause - Measured reason the automatic handoff cannot run.
|
|
820
|
-
* @returns {Error} Refusal naming both versions and the exact repair command.
|
|
821
|
-
*/
|
|
822
|
-
guardianVersionHandoffRefusal(guardianVersion, cause) {
|
|
823
|
-
const configArgument = this.configPath ? ` --config ${this.configPath}` : ""
|
|
824
|
-
|
|
825
|
-
return new Error(`The persistent Rollbridge guardian is version ${guardianVersion ?? "unknown"} while this daemon runs rollbridge ${this.runtime?.version ?? "unknown"}; ${cause}. Run "rollbridge shutdown${configArgument}" and then "rollbridge ensure-daemon${configArgument}" to replace the supervisor with this Rollbridge installation.`)
|
|
826
|
-
}
|
|
827
|
-
|
|
828
767
|
/** Requires the persistent guardian protocol which can restart an accepted daemon. */
|
|
829
768
|
async assertGuardianDaemonRecoveryCapability() {
|
|
830
769
|
if (!this.guardian) throw new Error("Owner recovery capability check requires the durable process guardian")
|
|
@@ -833,18 +772,10 @@ export default class RollbridgeDaemon {
|
|
|
833
772
|
try {
|
|
834
773
|
capabilities = await this.guardian.capabilities()
|
|
835
774
|
} catch (error) {
|
|
836
|
-
if (!(error instanceof Error) || !
|
|
775
|
+
if (!(error instanceof Error) || !["Guardian capabilities requires a process key", "Unknown guardian command: capabilities"].includes(error.message)) throw error
|
|
837
776
|
throw new Error("The persistent Rollbridge guardian predates daemon recovery. Perform one explicit clean Rollbridge shutdown and restart before retrying this package upgrade.", {cause: error})
|
|
838
777
|
}
|
|
839
778
|
if (capabilities.daemonRecovery !== 1) throw new Error("The persistent Rollbridge guardian does not support the required daemon recovery protocol")
|
|
840
|
-
const clientVersion = this.runtime?.version
|
|
841
|
-
const guardianVersion = capabilities.guardianVersion
|
|
842
|
-
|
|
843
|
-
if (!clientVersion) return
|
|
844
|
-
if (!guardianVersion) throw this.guardianVersionHandoffRefusal(guardianVersion, "its version cannot be attested, so no handoff can be authorised")
|
|
845
|
-
if (guardianVersion !== clientVersion) {
|
|
846
|
-
throw this.guardianVersionHandoffRefusal(guardianVersion, "this startup path has no incumbent daemon to hand guardian ownership from")
|
|
847
|
-
}
|
|
848
779
|
}
|
|
849
780
|
|
|
850
781
|
/**
|
|
@@ -965,15 +896,13 @@ export default class RollbridgeDaemon {
|
|
|
965
896
|
}
|
|
966
897
|
|
|
967
898
|
/**
|
|
968
|
-
* Authenticates and prepares the disruptive
|
|
969
|
-
*
|
|
970
|
-
* @param {{guardianVersion?: string, persisted: OwnerRecoverySnapshot, persistedAuthority: {configDigest: string, runtime: import("./daemon-runtime.js").DaemonRuntimeIdentity | null}, protocol: GuardianHandoffProtocol}} options - Bridge evidence and why the bridge is required.
|
|
899
|
+
* Authenticates and prepares the one-time disruptive bridge for an exact legacy guardian protocol.
|
|
900
|
+
* @param {{persisted: OwnerRecoverySnapshot, persistedAuthority: {configDigest: string, runtime: import("./daemon-runtime.js").DaemonRuntimeIdentity | null}}} options - Legacy evidence.
|
|
971
901
|
* @returns {Promise<LegacyOwnerBridge>} Prepared bridge.
|
|
972
902
|
*/
|
|
973
|
-
async prepareLegacyOwnerReplacement({
|
|
903
|
+
async prepareLegacyOwnerReplacement({persisted, persistedAuthority}) {
|
|
974
904
|
const legacyProcessKey = ownerSnapshotProcessKeys(persisted, persisted.singletonReleaseIds)[0]
|
|
975
905
|
const legacyGuardian = this.guardian
|
|
976
|
-
const versionHandoff = protocol === "guardian-version-drift"
|
|
977
906
|
let legacyPrepared
|
|
978
907
|
|
|
979
908
|
if (!legacyProcessKey) throw new Error("Legacy disruptive owner replacement requires an exact guardian-owned process registration in durable state")
|
|
@@ -1006,23 +935,12 @@ export default class RollbridgeDaemon {
|
|
|
1006
935
|
|
|
1007
936
|
try {
|
|
1008
937
|
if (persisted.recovery.configDigest !== this.ownerRecoveryConfigDigest()) {
|
|
1009
|
-
if (versionHandoff) throw this.guardianVersionHandoffRefusal(guardianVersion, "the handoff keeps the incumbent config identity unchanged and this deploy changes config authority")
|
|
1010
938
|
throw new Error("The one-time legacy guardian bridge requires the incumbent config identity unchanged; retry the config change after the bridge establishes split-3 authority")
|
|
1011
939
|
}
|
|
1012
|
-
if (!this.legacyIncumbentPid)
|
|
1013
|
-
if (versionHandoff) throw this.guardianVersionHandoffRefusal(guardianVersion, "ensure-daemon reported no incumbent daemon to hand guardian ownership from")
|
|
1014
|
-
throw new Error("The authenticated legacy guardian requires an exact incumbent PID from ensure-daemon for the disruptive bridge")
|
|
1015
|
-
}
|
|
940
|
+
if (!this.legacyIncumbentPid) throw new Error("The authenticated legacy guardian requires an exact incumbent PID from ensure-daemon for the disruptive bridge")
|
|
1016
941
|
if (!this.guardianIdentity?.pid) throw new Error("The authenticated legacy guardian state is missing its exact guardian PID")
|
|
1017
942
|
await verifyLegacyGuardianProcess(this.guardianIdentity.pid, this.guardianIdentity.socketPath)
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
try {
|
|
1021
|
-
incumbentStartTime = await verifyLegacyDaemonProcess(this.legacyIncumbentPid, this.configPath, persisted.control.path)
|
|
1022
|
-
} catch (error) {
|
|
1023
|
-
if (!versionHandoff) throw error
|
|
1024
|
-
throw this.guardianVersionHandoffRefusal(guardianVersion, `the incumbent daemon PID ${this.legacyIncumbentPid} is not the attested owner: ${error instanceof Error ? error.message : String(error)}`)
|
|
1025
|
-
}
|
|
943
|
+
const incumbentStartTime = await verifyLegacyDaemonProcess(this.legacyIncumbentPid, this.configPath, persisted.control.path)
|
|
1026
944
|
if (legacyPrepared) assertLegacyPrivateOwnerState(legacyPrepared.ownerState, persisted, persistedAuthority)
|
|
1027
945
|
try {
|
|
1028
946
|
incumbentControl = await openControlSession(persisted.control.path)
|
|
@@ -1049,9 +967,7 @@ export default class RollbridgeDaemon {
|
|
|
1049
967
|
|
|
1050
968
|
assertLegacyGuardianInventoryMembership(legacyInventory, /** @type {PrivateOwnerState} */ (ownerState))
|
|
1051
969
|
const upgradedIdentity = /** @type {{pid?: number, socketPath: string, token: string}} */ ({
|
|
1052
|
-
socketPath:
|
|
1053
|
-
? `${this.statePath}.guardian-${this.runtime?.version}.sock`
|
|
1054
|
-
: `${this.statePath}.split3-guardian.sock`),
|
|
970
|
+
socketPath: `${this.statePath}.split3-guardian.sock`,
|
|
1055
971
|
token: crypto.randomBytes(32).toString("hex")
|
|
1056
972
|
})
|
|
1057
973
|
|
|
@@ -1066,15 +982,12 @@ export default class RollbridgeDaemon {
|
|
|
1066
982
|
recovery: {...persisted.recovery, guardian: upgradedIdentity}
|
|
1067
983
|
})
|
|
1068
984
|
|
|
1069
|
-
this.logger("
|
|
985
|
+
this.logger("legacy owner replacement bridge prepared", {
|
|
1070
986
|
guardianPid: upgradedIdentity.pid ?? null,
|
|
1071
|
-
guardianVersion: guardianVersion ?? "unknown",
|
|
1072
987
|
incumbentPid: this.legacyIncumbentPid,
|
|
1073
|
-
|
|
1074
|
-
replacementId: prepared.replacementId,
|
|
1075
|
-
runtimeVersion: this.runtime?.version ?? "unknown"
|
|
988
|
+
replacementId: prepared.replacementId
|
|
1076
989
|
})
|
|
1077
|
-
return {boundaryCrossed: false,
|
|
990
|
+
return {boundaryCrossed: false, incumbentControl, incumbentStartTime, legacyGuardian, legacyInventory, legacyPrepared, legacySnapshot: persisted, prepared, recoverySnapshot}
|
|
1078
991
|
} catch (upgradeError) {
|
|
1079
992
|
incumbentControl?.close()
|
|
1080
993
|
let cleanupError
|
|
@@ -1154,20 +1067,10 @@ export default class RollbridgeDaemon {
|
|
|
1154
1067
|
|
|
1155
1068
|
if (staged.committed) throw new Error("Legacy guardian committed replacement before the authenticated disruptive boundary")
|
|
1156
1069
|
}
|
|
1157
|
-
|
|
1158
|
-
? `retained guardian ${bridge.guardianVersion ?? "unknown"} predates this daemon runtime ${this.runtime?.version ?? "unknown"}`
|
|
1159
|
-
: "retained guardian and daemon lacked atomic replacement protocol"
|
|
1160
|
-
|
|
1161
|
-
this.logger("guardian upgrade bridge crossed", {
|
|
1070
|
+
this.logger("legacy owner replacement disruptive boundary", {
|
|
1162
1071
|
disruptive: true,
|
|
1163
|
-
guardianVersion: bridge.guardianVersion ?? "unknown",
|
|
1164
1072
|
incumbentPid: this.legacyIncumbentPid,
|
|
1165
|
-
|
|
1166
|
-
reason: boundaryReason,
|
|
1167
|
-
// The retained guardian keeps its own children; the boundary retires only the incumbent daemon.
|
|
1168
|
-
retainedGuardianPid: bridge.legacyGuardian.pid ?? null,
|
|
1169
|
-
retainedGuardianProcesses: bridge.legacyInventory.length,
|
|
1170
|
-
runtimeVersion: this.runtime?.version ?? "unknown"
|
|
1073
|
+
reason: "retained guardian and daemon lacked atomic replacement protocol"
|
|
1171
1074
|
})
|
|
1172
1075
|
await this.guardian.beginLegacyOwnerClaim(
|
|
1173
1076
|
bridge.prepared.replacementId,
|
|
@@ -1183,8 +1086,8 @@ export default class RollbridgeDaemon {
|
|
|
1183
1086
|
await this.guardian.completeLegacyOwnerClaim(bridge.prepared.replacementId)
|
|
1184
1087
|
this.ownerTransition = /** @type {OwnerTransition} */ ({
|
|
1185
1088
|
disruptive: true,
|
|
1186
|
-
mode:
|
|
1187
|
-
reason:
|
|
1089
|
+
mode: "legacy-first-upgrade",
|
|
1090
|
+
reason: "retained guardian and daemon lacked atomic replacement protocol"
|
|
1188
1091
|
})
|
|
1189
1092
|
}
|
|
1190
1093
|
|
|
@@ -1486,7 +1389,7 @@ export default class RollbridgeDaemon {
|
|
|
1486
1389
|
releaseId: stringOrUndefined(data.releaseId),
|
|
1487
1390
|
releasePath: requiredString(data.releasePath, "releasePath"),
|
|
1488
1391
|
revision: stringOrUndefined(data.revision)
|
|
1489
|
-
})
|
|
1392
|
+
}))
|
|
1490
1393
|
}
|
|
1491
1394
|
|
|
1492
1395
|
if (commandName === "recover-generation-transition") {
|
|
@@ -1496,22 +1399,11 @@ export default class RollbridgeDaemon {
|
|
|
1496
1399
|
releaseId: requiredString(data.releaseId, "releaseId"),
|
|
1497
1400
|
releasePath: requiredString(data.releasePath, "releasePath"),
|
|
1498
1401
|
revision: requiredString(data.revision, "revision")
|
|
1499
|
-
})
|
|
1402
|
+
}))
|
|
1500
1403
|
}
|
|
1501
1404
|
|
|
1502
1405
|
if (commandName === "status") {
|
|
1503
|
-
|
|
1504
|
-
let guardian
|
|
1505
|
-
|
|
1506
|
-
try {
|
|
1507
|
-
const diagnostics = await this.guardian?.guardianStatus()
|
|
1508
|
-
|
|
1509
|
-
guardian = diagnostics ? {guardianPid: diagnostics.guardianPid, ownerMutation: diagnostics.ownerMutation, version: diagnostics.version} : {guardianPid: null, ownerMutation: null, version: null}
|
|
1510
|
-
} catch (error) {
|
|
1511
|
-
guardian = {error: error instanceof Error ? error.message : String(error), guardianPid: this.guardianIdentity?.pid ?? null, ownerMutation: null, version: null}
|
|
1512
|
-
}
|
|
1513
|
-
|
|
1514
|
-
return {...status, guardian}
|
|
1406
|
+
return this.status({includeLogs: statusIncludeLogs(data.includeLogs)})
|
|
1515
1407
|
}
|
|
1516
1408
|
|
|
1517
1409
|
if (commandName === "events") {
|
|
@@ -1522,18 +1414,18 @@ export default class RollbridgeDaemon {
|
|
|
1522
1414
|
return await this.executeOwnerMutation("stop", async () => {
|
|
1523
1415
|
await this.stopRelease(stringOrUndefined(data.releaseId))
|
|
1524
1416
|
return this.status()
|
|
1525
|
-
}
|
|
1417
|
+
})
|
|
1526
1418
|
}
|
|
1527
1419
|
|
|
1528
1420
|
if (commandName === "restart") {
|
|
1529
1421
|
return await this.executeOwnerMutation("restart", async () => await this.restartProcesses({
|
|
1530
1422
|
policy: stringOrUndefined(data.policy),
|
|
1531
1423
|
processId: stringOrUndefined(data.processId)
|
|
1532
|
-
})
|
|
1424
|
+
}))
|
|
1533
1425
|
}
|
|
1534
1426
|
|
|
1535
1427
|
if (commandName === "rollback") {
|
|
1536
|
-
return await this.executeOwnerMutation("rollback", async () => await this.rollback({releaseId: stringOrUndefined(data.releaseId)})
|
|
1428
|
+
return await this.executeOwnerMutation("rollback", async () => await this.rollback({releaseId: stringOrUndefined(data.releaseId)}))
|
|
1537
1429
|
}
|
|
1538
1430
|
|
|
1539
1431
|
if (commandName === "shutdown") {
|
|
@@ -1597,13 +1489,11 @@ export default class RollbridgeDaemon {
|
|
|
1597
1489
|
* Runs one owner mutation under the guardian's prepare/commit exclusion fence.
|
|
1598
1490
|
* @param {string} operation - Mutation diagnostic name.
|
|
1599
1491
|
* @param {() => Promise<Record<string, JsonValue>>} callback - Mutating operation.
|
|
1600
|
-
* @param {{waitingFor?: string}} [options] - What this mutation blocks on, so a refused
|
|
1601
|
-
* later mutation names the real blocker instead of just "in progress".
|
|
1602
1492
|
* @returns {Promise<Record<string, JsonValue>>} Operation result.
|
|
1603
1493
|
*/
|
|
1604
|
-
async executeOwnerMutation(operation, callback
|
|
1494
|
+
async executeOwnerMutation(operation, callback) {
|
|
1605
1495
|
if (!this.guardian) return await callback()
|
|
1606
|
-
const mutationId = await this.guardian.beginOwnerMutation(operation
|
|
1496
|
+
const mutationId = await this.guardian.beginOwnerMutation(operation)
|
|
1607
1497
|
let result
|
|
1608
1498
|
let operationError
|
|
1609
1499
|
|
|
@@ -1739,46 +1629,6 @@ export default class RollbridgeDaemon {
|
|
|
1739
1629
|
await this.updateGenerationTransition("restoring_committed")
|
|
1740
1630
|
return await this.resumeGenerationTransition()
|
|
1741
1631
|
}
|
|
1742
|
-
await this.escalateWedgedGenerations(nextConfig, newReleaseId)
|
|
1743
|
-
// A release id that is still retained by a generation holding live work cannot be started
|
|
1744
|
-
// a second time: the candidate's processes re-register the guardian keys that the retained
|
|
1745
|
-
// generation's own processes still hold, so the deploy aborts partway through `start()`
|
|
1746
|
-
// with a bare `Guardian provenance mismatch` — after the caller has committed to this
|
|
1747
|
-
// revision and, for a restore, after the deployment pointer has already moved. Refusing
|
|
1748
|
-
// from the retained record before any process starts keeps the incumbent serving, names the
|
|
1749
|
-
// release the guardian reports as active, and leaves the retained generation intact for
|
|
1750
|
-
// `rollbridge status` to reconcile against. Journaled transition resume above is exempt:
|
|
1751
|
-
// it re-drives the transaction that already owns this release id.
|
|
1752
|
-
//
|
|
1753
|
-
// The escalation above runs first on purpose: it is what clears the residue that made this
|
|
1754
|
-
// guard refuse the measured 2026-11-15 and 2026-11-16 deploys. A wedge that escalation can
|
|
1755
|
-
// prove is kill-safe stops holding its keys and the handoff proceeds; a generation that
|
|
1756
|
-
// still holds live work is not killed and is named here instead of aborting mid-`start()`.
|
|
1757
|
-
const heldGeneration = this.retainedGenerationHoldingLiveWork(newReleaseId)
|
|
1758
|
-
|
|
1759
|
-
if (heldGeneration) {
|
|
1760
|
-
const activeReleaseId = this.activeRelease?.releaseId ?? null
|
|
1761
|
-
|
|
1762
|
-
this.logger("deploy refused: retained generation still holds live processes", {
|
|
1763
|
-
activeReleaseId,
|
|
1764
|
-
heldProcesses: heldGeneration.processes,
|
|
1765
|
-
releaseId: newReleaseId,
|
|
1766
|
-
retainedState: heldGeneration.state
|
|
1767
|
-
})
|
|
1768
|
-
|
|
1769
|
-
throw new Error(`Deploy refused before activation: release "${newReleaseId}" is retained in state ${heldGeneration.state} while ${heldGeneration.count} process${heldGeneration.count === 1 ? "" : "es"} of that generation still hold live work (${heldGeneration.hold.join("; ")}). Activating that release id again would re-register those processes under a new provenance and abort with a guardian provenance mismatch. The guardian's active release is ${activeReleaseId ? `"${activeReleaseId}"` : "none"}; deploy a new release id or reconcile the deployment pointer with \`rollbridge status\`.`)
|
|
1770
|
-
}
|
|
1771
|
-
const admission = this.evaluateGenerationAdmission(newReleaseId, nextConfig)
|
|
1772
|
-
|
|
1773
|
-
if (!admission.allowed) {
|
|
1774
|
-
this.logger("deploy refused by draining-generation bound", {
|
|
1775
|
-
candidateReleaseId: newReleaseId,
|
|
1776
|
-
draining: admission.draining.map((group) => ({releaseId: group.releaseId, status: group.status})),
|
|
1777
|
-
limit: nextConfig.releaseRetention.maxConcurrentDrainingReleaseGroups,
|
|
1778
|
-
wouldBeDrainingCount: admission.wouldBeDrainingCount
|
|
1779
|
-
})
|
|
1780
|
-
throw new Error(/** @type {string} */ (admission.reason))
|
|
1781
|
-
}
|
|
1782
1632
|
const release = new ReleaseGroup({
|
|
1783
1633
|
config: nextConfig,
|
|
1784
1634
|
logger: this.logger,
|
|
@@ -1835,79 +1685,6 @@ export default class RollbridgeDaemon {
|
|
|
1835
1685
|
return await this.resumeGenerationTransition()
|
|
1836
1686
|
}
|
|
1837
1687
|
|
|
1838
|
-
/**
|
|
1839
|
-
* Applies the draining-generation bound before a handoff.
|
|
1840
|
-
*
|
|
1841
|
-
* Without this, every deploy retires one more generation and the backlog grows
|
|
1842
|
-
* monotonically whenever a drain outlasts the deploy cadence; a consumer that checks the
|
|
1843
|
-
* count after activation reports the growth it cannot undo. Refusing here names the
|
|
1844
|
-
* draining groups and whether each is still progressing or wedged, before any process of
|
|
1845
|
-
* the candidate is started.
|
|
1846
|
-
* @param {string} candidateReleaseId - Release the handoff would activate.
|
|
1847
|
-
* @param {import("./config.js").RollbridgeConfig} config - Config governing the handoff.
|
|
1848
|
-
* @returns {import("./drain-policy.js").GenerationAdmission} Admission decision and the groups behind it.
|
|
1849
|
-
*/
|
|
1850
|
-
evaluateGenerationAdmission(candidateReleaseId, config) {
|
|
1851
|
-
const draining = [...this.releases.values()]
|
|
1852
|
-
.filter((release) => release.state === "draining")
|
|
1853
|
-
.map((release) => {
|
|
1854
|
-
const drain = release.drainStatus()
|
|
1855
|
-
|
|
1856
|
-
return {reason: drain?.reason ?? "no liveness observation recorded", releaseId: release.releaseId, status: drain?.status ?? "unproven"}
|
|
1857
|
-
})
|
|
1858
|
-
|
|
1859
|
-
return admitGenerationRelease({
|
|
1860
|
-
candidateReleaseId,
|
|
1861
|
-
draining,
|
|
1862
|
-
incumbentWillDrain: this.activeRelease !== undefined,
|
|
1863
|
-
maxConcurrentDrainingReleaseGroups: config.releaseRetention.maxConcurrentDrainingReleaseGroups
|
|
1864
|
-
})
|
|
1865
|
-
}
|
|
1866
|
-
|
|
1867
|
-
/**
|
|
1868
|
-
* Reports whether a retained release group still holds live work that a re-activation would collide with.
|
|
1869
|
-
*
|
|
1870
|
-
* `stopped` is not the same claim as `drained`: the guardian's provenance fence only lets a
|
|
1871
|
-
* process key be re-registered once the record that holds it is gone. A generation recorded
|
|
1872
|
-
* as drained while its process group is still alive — the shape an indefinite worker drain
|
|
1873
|
-
* produces — still holds its keys, so its release id is not available for a restore.
|
|
1874
|
-
* @param {string} releaseId - Release id a handoff would activate.
|
|
1875
|
-
* @returns {HeldGenerationHold | undefined} The live hold, or undefined when the id is free.
|
|
1876
|
-
*/
|
|
1877
|
-
retainedGenerationHoldingLiveWork(releaseId) {
|
|
1878
|
-
const retained = this.releases.get(releaseId)
|
|
1879
|
-
|
|
1880
|
-
if (!retained || retained === this.activeRelease) return undefined
|
|
1881
|
-
|
|
1882
|
-
return heldGenerationHold(retained)
|
|
1883
|
-
}
|
|
1884
|
-
|
|
1885
|
-
/**
|
|
1886
|
-
* Gives every retiring generation the escalation decision its own retirement left open,
|
|
1887
|
-
* before the handoff is judged.
|
|
1888
|
-
*
|
|
1889
|
-
* A deploy is the event that makes a wedged generation expensive: until something tries to
|
|
1890
|
-
* activate a successor, a retired release holding a live process group costs nothing
|
|
1891
|
-
* observable. Acting here means the observation that proves the wedge and the action on it
|
|
1892
|
-
* come from the same read the admission bound already performs, so no sampler has to run
|
|
1893
|
-
* alongside the daemon. Generations that already recorded `release drained` are included:
|
|
1894
|
-
* their retirement finished, and the process group they left behind is the residue that
|
|
1895
|
-
* blocked the measured 2026-11-15 deploys.
|
|
1896
|
-
* @param {import("./config.js").RollbridgeConfig} config - Config governing the handoff.
|
|
1897
|
-
* @param {string} candidateReleaseId - Release the handoff would activate.
|
|
1898
|
-
* @returns {Promise<void>} Resolves once every retiring generation has been decided.
|
|
1899
|
-
*/
|
|
1900
|
-
async escalateWedgedGenerations(config, candidateReleaseId) {
|
|
1901
|
-
if (!config.releaseRetention.escalateWedgedGenerations) return
|
|
1902
|
-
|
|
1903
|
-
for (const release of this.releases.values()) {
|
|
1904
|
-
if (release.releaseId === candidateReleaseId) continue
|
|
1905
|
-
if (release.state !== "draining" && release.state !== "stopped" && release.state !== "failed") continue
|
|
1906
|
-
|
|
1907
|
-
await release.escalateWedgedGeneration({reason: `deploy of ${candidateReleaseId}`})
|
|
1908
|
-
}
|
|
1909
|
-
}
|
|
1910
|
-
|
|
1911
1688
|
/**
|
|
1912
1689
|
* Continues one exact durable generation transition without an internal retry loop.
|
|
1913
1690
|
* @returns {Promise<Record<string, JsonValue>>} Deploy result after commit.
|
|
@@ -1929,9 +1706,10 @@ export default class RollbridgeDaemon {
|
|
|
1929
1706
|
let retirementFailure = activationLifecycle ? undefined : previousRelease?.retirementError
|
|
1930
1707
|
|
|
1931
1708
|
if (transition.phase === "candidate_ready") {
|
|
1932
|
-
//
|
|
1933
|
-
//
|
|
1934
|
-
//
|
|
1709
|
+
// Candidate processes are already started and health-checked. Keep the incumbent
|
|
1710
|
+
// serving until the candidate has acknowledged activation and traffic has switched;
|
|
1711
|
+
// retiring it here can stop the only live web/jobs generation before its successor
|
|
1712
|
+
// carries traffic (STOE-2269).
|
|
1935
1713
|
await this.updateGenerationTransition("activating_candidate")
|
|
1936
1714
|
}
|
|
1937
1715
|
|
|
@@ -2006,19 +1784,6 @@ export default class RollbridgeDaemon {
|
|
|
2006
1784
|
|
|
2007
1785
|
if (transition.phase === "committed_pending") {
|
|
2008
1786
|
await this.refreshServiceDefinitions(release)
|
|
2009
|
-
if (previousRelease?.state === "active") {
|
|
2010
|
-
const retirementConfig = previousRelease.preserveConfigOnRetirement ? previousRelease.config : release.config
|
|
2011
|
-
|
|
2012
|
-
// Traffic is already on the healthy candidate. Retirement must not hold
|
|
2013
|
-
// the deploy mutation or its owner fence; its own bounded drain reports
|
|
2014
|
-
// failures and escalates any process that misses the retirement budget.
|
|
2015
|
-
void previousRelease.beginRetirement(retirementConfig)
|
|
2016
|
-
.catch((error) => {
|
|
2017
|
-
retirementFailure = previousRelease.retirementError ?? (error instanceof Error ? error.message : String(error))
|
|
2018
|
-
this.logger("release retirement quiescence failed", {error: retirementFailure, releaseId: previousRelease.releaseId})
|
|
2019
|
-
})
|
|
2020
|
-
.then(() => this.drainAndPrune(previousRelease, retirementConfig))
|
|
2021
|
-
}
|
|
2022
1787
|
await this.checkpointGenerationTransition()
|
|
2023
1788
|
try {
|
|
2024
1789
|
await this.replaceSingletons(release)
|
|
@@ -2026,12 +1791,29 @@ export default class RollbridgeDaemon {
|
|
|
2026
1791
|
await this.failGenerationTransition(error instanceof Error ? error.message : String(error))
|
|
2027
1792
|
throw error
|
|
2028
1793
|
}
|
|
1794
|
+
if (previousRelease) {
|
|
1795
|
+
const retirementConfig = previousRelease.preserveConfigOnRetirement ? previousRelease.config : release.config
|
|
1796
|
+
|
|
1797
|
+
// Traffic is already on the candidate and singleton ownership has been
|
|
1798
|
+
// transferred. Retirement is deliberately detached so a slow old generation
|
|
1799
|
+
// cannot hold the committed deploy open or race guardian-owned replacement.
|
|
1800
|
+
void this.drainAndPrune(previousRelease, retirementConfig)
|
|
1801
|
+
}
|
|
2029
1802
|
transition.phase = "committed"
|
|
2030
1803
|
transition.error = undefined
|
|
2031
1804
|
transition.journalRevision = (transition.journalRevision ?? 0) + 1
|
|
2032
1805
|
transition.updatedAt = new Date().toISOString()
|
|
2033
1806
|
this.pruneStoppedReleases()
|
|
2034
1807
|
await this.checkpointGenerationTransition()
|
|
1808
|
+
if (previousRelease) {
|
|
1809
|
+
const retirementConfig = !activationLifecycle && previousRelease.preserveConfigOnRetirement ? previousRelease.config : release.config
|
|
1810
|
+
|
|
1811
|
+
// Traffic is already on the healthy candidate. Retirement is deliberately detached
|
|
1812
|
+
// from the owner mutation so a slow quiet/drain hook can never hold the deploy fence
|
|
1813
|
+
// or make the active successor unavailable. drainAndStop applies the bounded force
|
|
1814
|
+
// deadline and records any retirement failure on the old generation.
|
|
1815
|
+
void this.drainAndPrune(previousRelease, retirementConfig)
|
|
1816
|
+
}
|
|
2035
1817
|
}
|
|
2036
1818
|
|
|
2037
1819
|
return {
|
|
@@ -2061,8 +1843,6 @@ export default class RollbridgeDaemon {
|
|
|
2061
1843
|
if (!candidate) throw new Error(`Generation transition candidate ${transition.candidateReleaseId} is not retained`)
|
|
2062
1844
|
if (!previous) throw new Error(`Generation transition previous release ${transition.previousReleaseId} is not retained`)
|
|
2063
1845
|
if (this.activeRelease !== previous) throw new Error(`Previous release ${previous.releaseId} is not the authoritative proxy target`)
|
|
2064
|
-
/** @type {Promise<void> | undefined} */
|
|
2065
|
-
let candidateDrain
|
|
2066
1846
|
|
|
2067
1847
|
if (transition.phase === "activating_candidate") {
|
|
2068
1848
|
if (!transition.error && !transition.activationError) throw new Error("Candidate activation has no recorded failure to compensate")
|
|
@@ -2094,15 +1874,6 @@ export default class RollbridgeDaemon {
|
|
|
2094
1874
|
transition.compensationError = undefined
|
|
2095
1875
|
await this.updateGenerationTransition("restoring_previous")
|
|
2096
1876
|
this.logger("release generation compensation candidate retired", {releaseId: candidate.releaseId})
|
|
2097
|
-
|
|
2098
|
-
// The candidate's bounded drain starts here, before the restoration attempt. The only other
|
|
2099
|
-
// compensation drain runs after the success path, so when incumbent restoration throws the
|
|
2100
|
-
// candidate is left `draining` with a live process: measured 2026-11-17, candidate
|
|
2101
|
-
// `20261117063703` still held `web` pid 3201928 at 07:30Z after its compensation threw at
|
|
2102
|
-
// 06:42:31Z, and reached `release drained` only when a later compensation reached the
|
|
2103
|
-
// success path at 08:47:00Z. A retained generation holding live work is what refuses every
|
|
2104
|
-
// later deploy, and a failed candidate is never a restoration target.
|
|
2105
|
-
candidateDrain = this.drainAndPrune(candidate, candidate.config)
|
|
2106
1877
|
}
|
|
2107
1878
|
|
|
2108
1879
|
if (transition.phase === "restoring_previous") {
|
|
@@ -2114,12 +1885,10 @@ export default class RollbridgeDaemon {
|
|
|
2114
1885
|
|
|
2115
1886
|
transition.compensationError = failure
|
|
2116
1887
|
await this.failGenerationTransition(`Incumbent ${previous.releaseId} restoration failed: ${failure}`)
|
|
2117
|
-
await previous.stopProxyTraffic()
|
|
2118
1888
|
this.logger("release generation compensation restoration failed", {
|
|
2119
1889
|
activationError: transition.activationError,
|
|
2120
1890
|
error: failure,
|
|
2121
|
-
releaseId: previous.releaseId
|
|
2122
|
-
traffic: "unavailable"
|
|
1891
|
+
releaseId: previous.releaseId
|
|
2123
1892
|
})
|
|
2124
1893
|
throw new Error(`incumbent ${previous.releaseId} restoration failed: ${failure}`, {cause: error})
|
|
2125
1894
|
}
|
|
@@ -2158,7 +1927,7 @@ export default class RollbridgeDaemon {
|
|
|
2158
1927
|
this.logger("release generation compensation checkpoint failed", {error: failure, releaseId: candidate.releaseId})
|
|
2159
1928
|
throw new Error(`compensation checkpoint clear failed: ${failure}`, {cause: error})
|
|
2160
1929
|
}
|
|
2161
|
-
|
|
1930
|
+
void this.drainAndPrune(candidate, candidate.config)
|
|
2162
1931
|
this.logger("release generation compensation completed", result)
|
|
2163
1932
|
return result
|
|
2164
1933
|
}
|
|
@@ -2182,19 +1951,6 @@ export default class RollbridgeDaemon {
|
|
|
2182
1951
|
recoveryStatus: "already_recovered"
|
|
2183
1952
|
}
|
|
2184
1953
|
}
|
|
2185
|
-
// A committed journal describes the transition that already published the current
|
|
2186
|
-
// incumbent, not an in-flight transition to recover. When the recovery fence names
|
|
2187
|
-
// that live incumbent, the failed deploy never journaled its own transition, so the
|
|
2188
|
-
// only fence that describes the authoritative release is the completed one: recovery
|
|
2189
|
-
// is already done. In-flight phases keep the stale-fence guard failing closed.
|
|
2190
|
-
if (transition.phase === "committed" && transition.candidateReleaseId === identity.previousReleaseId && this.activeRelease?.releaseId === identity.previousReleaseId && this.releases.get(transition.candidateReleaseId) === this.activeRelease) {
|
|
2191
|
-
return {
|
|
2192
|
-
activeReleaseId: this.activeRelease.releaseId,
|
|
2193
|
-
candidateReleaseId: identity.releaseId,
|
|
2194
|
-
previousReleaseId: identity.previousReleaseId,
|
|
2195
|
-
recoveryStatus: "already_recovered"
|
|
2196
|
-
}
|
|
2197
|
-
}
|
|
2198
1954
|
if (transition.previousReleaseId !== identity.previousReleaseId) {
|
|
2199
1955
|
throw new Error(`Generation transition previous release is ${transition.previousReleaseId}; refusing stale recovery for ${identity.previousReleaseId}`)
|
|
2200
1956
|
}
|
|
@@ -2641,38 +2397,11 @@ export default class RollbridgeDaemon {
|
|
|
2641
2397
|
throw new Error(`Release "${target.releaseId}" is already active.`)
|
|
2642
2398
|
}
|
|
2643
2399
|
|
|
2644
|
-
// A generation recorded as `stopped` whose processes are still alive is the shape a
|
|
2645
|
-
// restore cannot honour: re-deploying that release id re-registers its guardian process
|
|
2646
|
-
// keys under a new provenance while the old processes still hold them, and the guardian
|
|
2647
|
-
// answers with a bare provenance mismatch. Use the guardian's own fence here so the
|
|
2648
|
-
// refusal is decided before anything is touched, and name the divergence — including a
|
|
2649
|
-
// process group that outlived the tracked process and is invisible as a pid — together
|
|
2650
|
-
// with the release the guardian actually reports as active, so the deployment tool can
|
|
2651
|
-
// reconcile its own current pointer instead of restoring a generation that already drained.
|
|
2652
|
-
// A target that is still draining is not refused here: stopping it is what makes its id
|
|
2653
|
-
// reusable, and the deploy below applies the same fence after that stop.
|
|
2654
|
-
const heldGeneration = target.state === "stopped" ? heldGenerationHold(target) : undefined
|
|
2655
|
-
|
|
2656
|
-
if (heldGeneration) {
|
|
2657
|
-
const activeReleaseId = this.activeRelease?.releaseId ?? null
|
|
2658
|
-
|
|
2659
|
-
this.logger("rollback refused: drained generation still holds live processes", {
|
|
2660
|
-
activeReleaseId,
|
|
2661
|
-
heldProcesses: heldGeneration.processes,
|
|
2662
|
-
releaseId: target.releaseId,
|
|
2663
|
-
revision: target.revision
|
|
2664
|
-
})
|
|
2665
|
-
|
|
2666
|
-
throw new Error(`Release "${target.releaseId}" is recorded as drained while ${heldGeneration.count} process${heldGeneration.count === 1 ? "" : "es"} of that generation still hold live work (${heldGeneration.hold.join("; ")}). Restoring it would re-register those processes under a new provenance and fail with a guardian provenance mismatch. The guardian's active release is ${activeReleaseId ? `"${activeReleaseId}"` : "none"}; reconcile the deployment pointer with \`rollbridge status\` before restoring.`)
|
|
2667
|
-
}
|
|
2668
|
-
|
|
2669
2400
|
// The target may still be draining a prior deploy (live processes). Stop it before the
|
|
2670
2401
|
// deploy below re-uses its id in this.releases, otherwise the still-running instance
|
|
2671
2402
|
// would be dropped from status/pruning/shutdown and could be orphaned.
|
|
2672
|
-
// Rollback is an owner mutation too, so this stop carries the same retirement bound as
|
|
2673
|
-
// `stop`: a process the supervisor must not kill must not be able to hold the fence.
|
|
2674
2403
|
if (target.state !== "stopped" && target.state !== "failed") {
|
|
2675
|
-
await target.stop(
|
|
2404
|
+
await target.stop()
|
|
2676
2405
|
}
|
|
2677
2406
|
|
|
2678
2407
|
this.logger("rollback starting", {releaseId: target.releaseId, releasePath: target.releasePath})
|
|
@@ -2893,7 +2622,7 @@ export default class RollbridgeDaemon {
|
|
|
2893
2622
|
|
|
2894
2623
|
/**
|
|
2895
2624
|
* @param {string | undefined} releaseId - Release id, or active release when omitted.
|
|
2896
|
-
* @returns {Promise<void>} Resolves when stopped
|
|
2625
|
+
* @returns {Promise<void>} Resolves when stopped.
|
|
2897
2626
|
*/
|
|
2898
2627
|
async stopRelease(releaseId) {
|
|
2899
2628
|
this.assertNoUnresolvedGenerationTransition("stop")
|
|
@@ -2902,10 +2631,7 @@ export default class RollbridgeDaemon {
|
|
|
2902
2631
|
if (!release) throw new Error(`Release not found: ${releaseId || "active"}`)
|
|
2903
2632
|
if (release === this.activeRelease) this.activeRelease = undefined
|
|
2904
2633
|
|
|
2905
|
-
|
|
2906
|
-
// supervisor must not kill holds the fence for the whole drain and every later deploy is
|
|
2907
|
-
// refused. The retirement bound is the same one the deploy path already uses.
|
|
2908
|
-
await release.stop({retirementTimeoutMs: this.config.proxy.nonBlockingDrainTimeoutMs})
|
|
2634
|
+
await release.stop()
|
|
2909
2635
|
this.logger("release stopped", {releaseId: release.releaseId})
|
|
2910
2636
|
this.pruneStoppedReleases()
|
|
2911
2637
|
this.persistState()
|
|
@@ -3337,42 +3063,6 @@ function statusIncludeLogs(value) {
|
|
|
3337
3063
|
return value
|
|
3338
3064
|
}
|
|
3339
3065
|
|
|
3340
|
-
/**
|
|
3341
|
-
* Names the live work a retained release group still holds, using the guardian's own fence.
|
|
3342
|
-
*
|
|
3343
|
-
* A process counts as holding work when the guardian would refuse to hand its key to a new
|
|
3344
|
-
* provenance: a running or stopping process, or a record already `stopped` whose process group
|
|
3345
|
-
* outlived it and is invisible as a pid.
|
|
3346
|
-
* @param {ReleaseGroup} release - Retained generation to inspect.
|
|
3347
|
-
* @returns {HeldGenerationHold | undefined} The hold and its log payload, or undefined when every process is gone.
|
|
3348
|
-
*/
|
|
3349
|
-
function heldGenerationHold(release) {
|
|
3350
|
-
const heldProcesses = release.status().processes.filter((processStatus) => !childRecordIsGone(processStatus))
|
|
3351
|
-
|
|
3352
|
-
if (heldProcesses.length === 0) return undefined
|
|
3353
|
-
|
|
3354
|
-
return {
|
|
3355
|
-
count: heldProcesses.length,
|
|
3356
|
-
hold: heldProcesses.map((processStatus) => {
|
|
3357
|
-
if (processStatus.heldProcessGroup) {
|
|
3358
|
-
const {liveMemberCount, pgid} = processStatus.heldProcessGroup
|
|
3359
|
-
const members = liveMemberCount === undefined ? "" : ` with ${liveMemberCount} live member(s)`
|
|
3360
|
-
|
|
3361
|
-
return `${processStatus.id}: process group ${pgid}${members} still alive after the tracked process exited`
|
|
3362
|
-
}
|
|
3363
|
-
|
|
3364
|
-
return `${processStatus.id}: state ${processStatus.state}, pid ${processStatus.pid ?? "unknown"}`
|
|
3365
|
-
}),
|
|
3366
|
-
processes: heldProcesses.map((processStatus) => ({
|
|
3367
|
-
heldProcessGroupPgid: processStatus.heldProcessGroup?.pgid ?? null,
|
|
3368
|
-
pid: processStatus.pid ?? null,
|
|
3369
|
-
processId: processStatus.id,
|
|
3370
|
-
state: processStatus.state
|
|
3371
|
-
})),
|
|
3372
|
-
state: release.state
|
|
3373
|
-
}
|
|
3374
|
-
}
|
|
3375
|
-
|
|
3376
3066
|
/**
|
|
3377
3067
|
* @param {PrivateOwnerState} transfer - Authenticated private incumbent state.
|
|
3378
3068
|
* @param {GenerationTransition} transition - Unresolved generation transition.
|
|
@@ -3433,15 +3123,6 @@ export function isLegacyGuardianPrepareDiagnostic(diagnostic) {
|
|
|
3433
3123
|
diagnostic === "Unknown guardian command: prepare-owner-replacement"
|
|
3434
3124
|
}
|
|
3435
3125
|
|
|
3436
|
-
/**
|
|
3437
|
-
* Reports the capability diagnostics that identify a guardian predating daemon recovery.
|
|
3438
|
-
* @param {string} diagnostic - Guardian error message.
|
|
3439
|
-
* @returns {boolean} Whether the guardian predates the daemon recovery capability.
|
|
3440
|
-
*/
|
|
3441
|
-
function isGuardianDaemonRecoveryDiagnostic(diagnostic) {
|
|
3442
|
-
return diagnostic === "Guardian capabilities requires a process key" || diagnostic === "Unknown guardian command: capabilities"
|
|
3443
|
-
}
|
|
3444
|
-
|
|
3445
3126
|
/**
|
|
3446
3127
|
* @param {DaemonStatus} snapshot - Serialized owner process snapshot.
|
|
3447
3128
|
* @param {Record<string, string>} [singletonReleaseIds] - Exact singleton owner releases.
|
|
@@ -3764,7 +3445,7 @@ function requiredNonNegativeInteger(value, key) {
|
|
|
3764
3445
|
/**
|
|
3765
3446
|
* Selects stopped releases to prune by the retention policy, keeping the most recent.
|
|
3766
3447
|
* @param {PrunableRelease[]} releases - Status of all tracked releases, in deploy order (oldest first).
|
|
3767
|
-
* @param {
|
|
3448
|
+
* @param {import("./config.js").ReleaseRetentionConfig} policy - Retention policy.
|
|
3768
3449
|
* @param {number} now - Current epoch milliseconds.
|
|
3769
3450
|
* @returns {string[]} Release ids to remove.
|
|
3770
3451
|
*/
|
|
@@ -3789,7 +3470,7 @@ export function releasesToPrune(releases, policy, now) {
|
|
|
3789
3470
|
}
|
|
3790
3471
|
|
|
3791
3472
|
/**
|
|
3792
|
-
* @typedef {{alive: boolean, application?: string, activeReleaseId?: string | null,
|
|
3473
|
+
* @typedef {{alive: boolean, application?: string, activeReleaseId?: string | null, proxy?: {host: string, port: number}}} ControlSocketInspection
|
|
3793
3474
|
*/
|
|
3794
3475
|
|
|
3795
3476
|
/**
|
|
@@ -3864,7 +3545,7 @@ export async function inspectControlSocket(socketPath, timeoutMs = 1000) {
|
|
|
3864
3545
|
/**
|
|
3865
3546
|
* Parses a control status response line into a Rollbridge identity, if it is one.
|
|
3866
3547
|
* @param {string} line - JSON response line.
|
|
3867
|
-
* @returns {{application: string, activeReleaseId: string | null,
|
|
3548
|
+
* @returns {{application: string, activeReleaseId: string | null, proxy: {host: string, port: number} | undefined} | undefined} Identity, or undefined when unrecognized.
|
|
3868
3549
|
*/
|
|
3869
3550
|
function parseControlStatus(line) {
|
|
3870
3551
|
/** @type {JsonValue} */
|
|
@@ -3881,8 +3562,5 @@ function parseControlStatus(line) {
|
|
|
3881
3562
|
|
|
3882
3563
|
const proxy = "proxy" in parsed && parsed.proxy && typeof parsed.proxy === "object" && !Array.isArray(parsed.proxy) && typeof parsed.proxy.host === "string" && typeof parsed.proxy.port === "number" ? {host: parsed.proxy.host, port: parsed.proxy.port} : undefined
|
|
3883
3564
|
|
|
3884
|
-
|
|
3885
|
-
const guardian = guardianValue && typeof guardianValue.version === "string" || guardianValue && guardianValue.version === null ? {ownerMutation: guardianValue.ownerMutation && typeof guardianValue.ownerMutation === "object" && !Array.isArray(guardianValue.ownerMutation) ? guardianValue.ownerMutation : null, version: guardianValue.version} : undefined
|
|
3886
|
-
|
|
3887
|
-
return {activeReleaseId: typeof parsed.activeReleaseId === "string" ? parsed.activeReleaseId : null, application: parsed.application, guardian, proxy}
|
|
3565
|
+
return {activeReleaseId: typeof parsed.activeReleaseId === "string" ? parsed.activeReleaseId : null, application: parsed.application, proxy}
|
|
3888
3566
|
}
|