rollbridge 0.1.48 → 0.1.54
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -0
- package/docs/cli.md +7 -1
- package/docs/generation-deployment-contract.md +9 -0
- package/package.json +1 -1
- package/src/cli.js +10 -2
- package/src/daemon.js +105 -13
- package/src/process-guardian.js +5 -1
- package/src/release-group.js +48 -1
- package/test/control-protocol.test.js +61 -1
- package/test/guardian-client.test.js +23 -0
- package/test/logs.test.js +7 -0
- package/test/owner-recovery.test.js +31 -3
- package/test/owner-replacement.test.js +1 -1
- package/test/rollbridge.test.js +16 -3
package/README.md
CHANGED
|
@@ -576,6 +576,7 @@ Inspect state:
|
|
|
576
576
|
|
|
577
577
|
```bash
|
|
578
578
|
rollbridge status --config rollbridge.js
|
|
579
|
+
rollbridge status --no-logs --config rollbridge.js
|
|
579
580
|
```
|
|
580
581
|
|
|
581
582
|
`status` reports each managed process's `state`, `pid`, recent `logs`, last
|
|
@@ -587,6 +588,10 @@ and why it last started (`lastStartReason`: `deploy`, `crash`, `manual`, or
|
|
|
587
588
|
`rssBytes`, `memoryRestarts`, `lastMemoryRestartAt`, and `children` (the sampled
|
|
588
589
|
process tree — each group member's `pid`, `command`, and `rssBytes`).
|
|
589
590
|
|
|
591
|
+
For machine lifecycle attestations that do not need captured process output,
|
|
592
|
+
pass `--no-logs`. It returns the same status projection while omitting only each
|
|
593
|
+
release, service, and singleton process's `logs` array.
|
|
594
|
+
|
|
590
595
|
Print the recent captured stdout/stderr per process (a one-shot snapshot of the
|
|
591
596
|
retained `outputLines`, not a live stream):
|
|
592
597
|
|
package/docs/cli.md
CHANGED
|
@@ -253,7 +253,7 @@ stays safe.
|
|
|
253
253
|
## `status`
|
|
254
254
|
|
|
255
255
|
```
|
|
256
|
-
rollbridge status [--config <path>]
|
|
256
|
+
rollbridge status [--config <path>] [--no-logs]
|
|
257
257
|
```
|
|
258
258
|
|
|
259
259
|
Prints the daemon status JSON: the active release id, the proxy address, and —
|
|
@@ -264,6 +264,12 @@ Memory-supervised processes also report `rssBytes`, `memoryRestarts`,
|
|
|
264
264
|
`lastMemoryRestartAt`, and `children` (the process tree: each group member's
|
|
265
265
|
`pid`, `command`, and `rssBytes`).
|
|
266
266
|
|
|
267
|
+
`--no-logs` omits only the captured `logs` array from release, service, and
|
|
268
|
+
singleton process statuses. Use it for bounded machine-readable lifecycle
|
|
269
|
+
attestations that do not need process output. The equivalent control request is
|
|
270
|
+
`{"command":"status","includeLogs":false}`; omitted `includeLogs` continues to
|
|
271
|
+
return the complete status payload.
|
|
272
|
+
|
|
267
273
|
`daemonRuntime` identifies the immutable Rollbridge runtime serving the proxy:
|
|
268
274
|
its runtime `format`, package `version`, content `digest`, and absolute `path`.
|
|
269
275
|
`ensure-daemon` uses this attestation before reusing a responsive daemon.
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# Generation deployment contract
|
|
2
|
+
|
|
3
|
+
Rollbridge treats web authority, jobs-generation authority, durable transition state, and old-release draining as separate concerns.
|
|
4
|
+
|
|
5
|
+
A successful activation-lifecycle deployment starts and health-checks the candidate, waits for the old generation's retirement/quiescence acknowledgement, activates the candidate jobs generation, switches web authority, checkpoints `committed`, and returns. The acknowledged old generation then continues draining and stopping asynchronously; deployment completion does not wait for that later work.
|
|
6
|
+
|
|
7
|
+
Failed candidates never receive traffic. Recovery is exact-authority and journaled; no operator should edit durable state. A degraded incumbent keeps web authority while jobs remain explicitly degraded until a fresh generation commits.
|
|
8
|
+
|
|
9
|
+
Persistent guardians may retain old processes, but new release processes must use current lifecycle semantics. Lifecycle command failures must retain bounded stdout/stderr and stage metadata. Runtime content digest is authoritative; package versions are compatibility metadata rather than unique code identity.
|
package/package.json
CHANGED
package/src/cli.js
CHANGED
|
@@ -41,6 +41,7 @@ export async function runCli(argv) {
|
|
|
41
41
|
.option("--boot-attestation <digest>", "Opaque bootstrap ownership attestation (requires the complete bootstrap release tuple)")
|
|
42
42
|
.option("--takeover-owner", "Boot and health-check before retiring the current external owner")
|
|
43
43
|
.option("--replace-owner", "Resume a prepared durable owner replacement")
|
|
44
|
+
.option("--reset-retired-owner", "Reset missing guardian identity from the exact committed bootstrap")
|
|
44
45
|
.addOption(new Option("--guardian-daemon-log-path <path>").hideHelp())
|
|
45
46
|
.addOption(new Option("--guardian-daemon-pid-path <path>").hideHelp())
|
|
46
47
|
.addOption(new Option("--guardian-daemon-start-timeout-ms <ms>").hideHelp())
|
|
@@ -88,6 +89,12 @@ export async function runCli(argv) {
|
|
|
88
89
|
process.once("SIGTERM", () => { void shutdown() })
|
|
89
90
|
|
|
90
91
|
if (options.takeoverOwner && (!bootstrap || !bootstrap.attestation)) throw new Error("Daemon --takeover-owner requires the complete bootstrap release tuple and --boot-attestation.")
|
|
92
|
+
if (options.resetRetiredOwner && (!bootstrap || options.takeoverOwner || options.replaceOwner)) throw new Error("Daemon --reset-retired-owner requires the complete bootstrap tuple and cannot be combined with owner takeover/replacement.")
|
|
93
|
+
|
|
94
|
+
if (options.resetRetiredOwner) {
|
|
95
|
+
startupPromise = daemon.resetRetiredOwnerRecovery()
|
|
96
|
+
await startupPromise
|
|
97
|
+
}
|
|
91
98
|
|
|
92
99
|
if (options.replaceOwner) {
|
|
93
100
|
if (bootstrap || options.takeoverOwner) throw new Error("Daemon --replace-owner cannot be combined with bootstrap takeover options.")
|
|
@@ -95,7 +102,7 @@ export async function runCli(argv) {
|
|
|
95
102
|
await startupPromise
|
|
96
103
|
}
|
|
97
104
|
|
|
98
|
-
if (!options.takeoverOwner && !options.replaceOwner) {
|
|
105
|
+
if (!options.takeoverOwner && !options.replaceOwner && !options.resetRetiredOwner) {
|
|
99
106
|
try {
|
|
100
107
|
startupPromise = daemon.start({exposeControl: !bootstrap})
|
|
101
108
|
await startupPromise
|
|
@@ -266,11 +273,12 @@ export async function runCli(argv) {
|
|
|
266
273
|
program
|
|
267
274
|
.command("status")
|
|
268
275
|
.option("-c, --config <path>", "Config file path (defaults to rollbridge.js)")
|
|
276
|
+
.option("--no-logs", "Omit captured stdout/stderr from process statuses")
|
|
269
277
|
.action(async (options) => {
|
|
270
278
|
const configPath = await resolveConfigPath(options.config)
|
|
271
279
|
const config = await loadConfig(configPath)
|
|
272
280
|
const response = await sendControlCommand({
|
|
273
|
-
command: {command: "status"},
|
|
281
|
+
command: {command: "status", ...(options.logs ? {} : {includeLogs: false})},
|
|
274
282
|
path: config.control.path
|
|
275
283
|
})
|
|
276
284
|
|
package/src/daemon.js
CHANGED
|
@@ -27,6 +27,10 @@ const STATE_PERSIST_INTERVAL_MS = 5000
|
|
|
27
27
|
* @typedef {"candidate_ready" | "retiring_previous" | "previous_retired" | "activating_candidate" | "restoring_previous" | "retiring_failed_candidate" | "degraded_active" | "committed_pending" | "committed" | "restoring_committed"} GenerationTransitionPhase
|
|
28
28
|
* @typedef {{activationError?: string, activationLifecycle?: boolean, candidateReleaseId: string, candidateReleasePath: string, candidateRevision: string, compensationError?: string, configDigest: string, degradedIncumbent?: boolean, error?: string, journalRevision?: number, phase: GenerationTransitionPhase, previousReleaseId: string | null, startedAt: string, updatedAt: string}} GenerationTransition
|
|
29
29
|
* @typedef {{activeReleaseId: string | null, application: string, bootstrap: BootstrapIdentity | undefined, control: import("./config.js").ControlConfig, daemonPid: number, daemonRuntime: import("./daemon-runtime.js").DaemonRuntimeIdentity | undefined, generationTransition?: GenerationTransition, ownerRecovery: {configDigest: string, ready: boolean} | undefined, ownerTransition?: OwnerTransition, orphans: {id: string, pid: number, releaseId: string | null}[], proxy: {host: string, port: number | undefined, upstreamHost: string}, releaseReferences: {releaseId: string, releasePath: string}[], releases: import("./release-group.js").ReleaseStatus[], services: ProcessStatus[], singletons: ProcessStatus[]}} DaemonStatus
|
|
30
|
+
* @typedef {Omit<import("./managed-process.js").ManagedProcessStatus, "logs">} ManagedProcessStatusWithoutLogs
|
|
31
|
+
* @typedef {Omit<import("./release-group.js").ReleaseStatus, "processes"> & {processes: ManagedProcessStatusWithoutLogs[]}} ReleaseStatusWithoutLogs
|
|
32
|
+
* @typedef {Omit<ProcessStatus, "process"> & {process: ManagedProcessStatusWithoutLogs}} ProcessStatusWithoutLogs
|
|
33
|
+
* @typedef {Omit<DaemonStatus, "releases" | "services" | "singletons"> & {releases: ReleaseStatusWithoutLogs[], services: ProcessStatusWithoutLogs[], singletons: ProcessStatusWithoutLogs[]}} DaemonStatusWithoutLogs
|
|
30
34
|
* @typedef {{configDigest: string, format: number, guardian: {pid?: number, socketPath: string, token: string}, reconnectGraceMs: number}} OwnerRecoveryMetadata
|
|
31
35
|
* @typedef {DaemonStatus & {recovery: OwnerRecoveryMetadata, serviceReleaseIds?: Record<string, string>, singletonReleaseIds?: Record<string, string>}} OwnerRecoverySnapshot
|
|
32
36
|
* @typedef {{authority: JsonValue, config: import("./config.js").RollbridgeConfig, listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>, listenerSourceId?: string, recovery?: {command: JsonValue, reconnectGraceMs: number, startupTimeoutMs: number}, releaseConfigs?: Record<string, import("./config.js").RollbridgeConfig>, retiredListenerHandoff?: number, serviceReleaseIds?: Record<string, string>, singletonReleaseIds?: Record<string, string>, snapshot: OwnerRecoverySnapshot}} PrivateOwnerState
|
|
@@ -186,6 +190,28 @@ export default class RollbridgeDaemon {
|
|
|
186
190
|
return true
|
|
187
191
|
}
|
|
188
192
|
|
|
193
|
+
/** Re-establishes durable guardian ownership from an exact committed external-owner bootstrap. */
|
|
194
|
+
async resetRetiredOwnerRecovery() {
|
|
195
|
+
if (!this.statePath || !this.config.ownerRecovery || !this.bootstrap) throw new Error("Retired owner reset requires ownerRecovery, statePath, and the exact bootstrap tuple")
|
|
196
|
+
const state = await readState(this.statePath)
|
|
197
|
+
const snapshot = state && typeof state === "object" && !Array.isArray(state) ? /** @type {OwnerRecoverySnapshot} */ (state) : undefined
|
|
198
|
+
const transition = snapshot?.generationTransition
|
|
199
|
+
const persistedBootstrap = snapshot?.bootstrap
|
|
200
|
+
const exactBootstrap = snapshot?.activeReleaseId === this.bootstrap.releaseId &&
|
|
201
|
+
persistedBootstrap?.releaseId === this.bootstrap.releaseId && persistedBootstrap?.releasePath === this.bootstrap.releasePath && persistedBootstrap?.revision === this.bootstrap.revision &&
|
|
202
|
+
transition?.phase === "committed" && transition.candidateReleaseId === this.bootstrap.releaseId && transition.candidateReleasePath === this.bootstrap.releasePath && transition.candidateRevision === this.bootstrap.revision
|
|
203
|
+
if (!snapshot || snapshot.recovery?.guardian || !exactBootstrap) throw new Error("Retired owner reset requires an exact committed bootstrap with missing guardian identity")
|
|
204
|
+
const control = await inspectControlSocket(this.config.control.path)
|
|
205
|
+
if (control.alive) throw new Error(`Retired owner reset refuses responsive control owner at ${this.config.control.path}`)
|
|
206
|
+
this.guardianIdentity = {socketPath: `${this.statePath}.guardian.sock`, token: crypto.randomBytes(32).toString("hex")}
|
|
207
|
+
this.guardian = new GuardianClient(this.guardianIdentity)
|
|
208
|
+
await this.guardian.launch()
|
|
209
|
+
this.guardianIdentity.pid = this.guardian.pid
|
|
210
|
+
this.watchOwnerReplacementEvents()
|
|
211
|
+
await this.guardian.claimOwner(this.config.ownerRecovery.reconnectGraceMs ?? 30000, this.ownerAuthority())
|
|
212
|
+
this.logger("retired owner guardian reset prepared", {guardianPid: this.guardianIdentity.pid ?? null, releaseId: this.bootstrap.releaseId})
|
|
213
|
+
}
|
|
214
|
+
|
|
189
215
|
/** Connects to the durable process guardian and reconstructs a matching persisted owner snapshot. */
|
|
190
216
|
async initializeOwnerRecovery() {
|
|
191
217
|
if (!this.statePath) throw new Error("ownerRecovery requires statePath")
|
|
@@ -1349,7 +1375,7 @@ export default class RollbridgeDaemon {
|
|
|
1349
1375
|
}
|
|
1350
1376
|
|
|
1351
1377
|
if (commandName === "status") {
|
|
1352
|
-
return this.status()
|
|
1378
|
+
return this.status({includeLogs: statusIncludeLogs(data.includeLogs)})
|
|
1353
1379
|
}
|
|
1354
1380
|
|
|
1355
1381
|
if (commandName === "events") {
|
|
@@ -1936,18 +1962,16 @@ export default class RollbridgeDaemon {
|
|
|
1936
1962
|
if (candidate.releasePath !== transition.candidateReleasePath || candidate.revision !== transition.candidateRevision || ownerConfigDigest(candidate.config) !== transition.configDigest) {
|
|
1937
1963
|
throw new Error(`Generation transition candidate ${candidate.releaseId} does not retain its exact path, revision, and config authority`)
|
|
1938
1964
|
}
|
|
1939
|
-
|
|
1965
|
+
const recoverableCandidateStates = transition.phase === "retiring_previous" ? ["starting", "draining", "stopped"] : ["draining", "stopped"]
|
|
1966
|
+
|
|
1967
|
+
if (!recoverableCandidateStates.includes(candidate.state)) throw new Error(`Failed candidate ${candidate.releaseId} must be retiring or retired before accepting a retired incumbent`)
|
|
1940
1968
|
if (this.activeRelease !== previous) throw new Error(`Previous release ${previous.releaseId} is not the authoritative proxy target`)
|
|
1941
1969
|
const activationConfig = previous.config.processes.find((processConfig) => processConfig.lifecycle.activateCommand !== undefined)
|
|
1942
1970
|
const coordinator = activationConfig ? previous.getProcesses(activationConfig.id)[0]?.process : undefined
|
|
1943
|
-
|
|
1944
|
-
if (!releaseOwnsLiveProxyTraffic(previous)) throw new Error(`Previous release ${previous.releaseId} no longer owns live proxy/web traffic`)
|
|
1945
|
-
await candidate.drainAndStop(candidate.config.proxy.drainTimeoutMs, candidate.config)
|
|
1946
|
-
const candidateProcesses = candidate.status().processes
|
|
1971
|
+
const missingRetiredCoordinator = !coordinator && /^Process .+ is not retained for reactivation$/u.test(retirementFailure ?? "")
|
|
1947
1972
|
|
|
1948
|
-
if (
|
|
1949
|
-
|
|
1950
|
-
}
|
|
1973
|
+
if ((!coordinator && !missingRetiredCoordinator) || (coordinator && coordinator.status().lifecycleRole !== "retired")) throw new Error(`Previous release ${previous.releaseId} does not retain a retired or terminally absent generation coordinator`)
|
|
1974
|
+
if (!releaseOwnsLiveProxyTraffic(previous)) throw new Error(`Previous release ${previous.releaseId} no longer owns live proxy/web traffic`)
|
|
1951
1975
|
if (this.activeRelease !== previous || !releaseOwnsLiveProxyTraffic(previous)) throw new Error(`Previous release ${previous.releaseId} lost live proxy/web authority during retired-incumbent recovery`)
|
|
1952
1976
|
const result = /** @type {Record<string, JsonValue>} */ ({
|
|
1953
1977
|
activeReleaseId: previous.releaseId,
|
|
@@ -1958,7 +1982,10 @@ export default class RollbridgeDaemon {
|
|
|
1958
1982
|
recoveryStatus: "retired_incumbent_accepted"
|
|
1959
1983
|
})
|
|
1960
1984
|
|
|
1961
|
-
if (transition.phase === "degraded_active")
|
|
1985
|
+
if (transition.phase === "degraded_active") {
|
|
1986
|
+
if (candidate.state !== "stopped") void this.drainAndPrune(candidate, candidate.config)
|
|
1987
|
+
return result
|
|
1988
|
+
}
|
|
1962
1989
|
const previousError = transition.error
|
|
1963
1990
|
const previousPhase = transition.phase
|
|
1964
1991
|
const previousJournalRevision = transition.journalRevision
|
|
@@ -1982,6 +2009,7 @@ export default class RollbridgeDaemon {
|
|
|
1982
2009
|
}
|
|
1983
2010
|
throw new Error(`retired-incumbent recovery checkpoint failed: ${error instanceof Error ? error.message : String(error)}`, {cause: error})
|
|
1984
2011
|
}
|
|
2012
|
+
if (candidate.state !== "stopped") void this.drainAndPrune(candidate, candidate.config)
|
|
1985
2013
|
this.logger("retired incumbent accepted as jobs-degraded", result)
|
|
1986
2014
|
return result
|
|
1987
2015
|
}
|
|
@@ -2886,8 +2914,25 @@ export default class RollbridgeDaemon {
|
|
|
2886
2914
|
return this.proxyPort
|
|
2887
2915
|
}
|
|
2888
2916
|
|
|
2889
|
-
/**
|
|
2890
|
-
|
|
2917
|
+
/**
|
|
2918
|
+
* @overload
|
|
2919
|
+
* @returns {DaemonStatus} Full status payload.
|
|
2920
|
+
*/
|
|
2921
|
+
/**
|
|
2922
|
+
* @overload
|
|
2923
|
+
* @param {{includeLogs: false}} options - Log-free status response options.
|
|
2924
|
+
* @returns {DaemonStatusWithoutLogs} Log-free status payload.
|
|
2925
|
+
*/
|
|
2926
|
+
/**
|
|
2927
|
+
* @overload
|
|
2928
|
+
* @param {{includeLogs: boolean}} options - Status response options.
|
|
2929
|
+
* @returns {DaemonStatus | DaemonStatusWithoutLogs} Status payload.
|
|
2930
|
+
*/
|
|
2931
|
+
/**
|
|
2932
|
+
* @param {{includeLogs?: boolean}} [options] - Status response options.
|
|
2933
|
+
* @returns {DaemonStatus | DaemonStatusWithoutLogs} Status payload.
|
|
2934
|
+
*/
|
|
2935
|
+
status({includeLogs = true} = {}) {
|
|
2891
2936
|
// Re-check liveness and prune the dead permanently, so the list self-clears as the operator
|
|
2892
2937
|
// stops the leftovers (e.g. via `rollbridge recover`). Pruning (not just filtering) matters:
|
|
2893
2938
|
// a cleared orphan must not reappear if the OS later recycles its pid for an unrelated process.
|
|
@@ -2897,7 +2942,7 @@ export default class RollbridgeDaemon {
|
|
|
2897
2942
|
const singletonOwnerReleaseIds = new Set(this.singletonReleaseIds.values())
|
|
2898
2943
|
const transitionReleaseIds = this.generationTransitionReleaseIds()
|
|
2899
2944
|
|
|
2900
|
-
|
|
2945
|
+
const status = {
|
|
2901
2946
|
activeReleaseId: this.activeRelease ? this.activeRelease.releaseId : null,
|
|
2902
2947
|
application: this.config.application,
|
|
2903
2948
|
bootstrap: this.bootstrap ? {...this.bootstrap} : undefined,
|
|
@@ -2928,6 +2973,8 @@ export default class RollbridgeDaemon {
|
|
|
2928
2973
|
process: processInstance.status()
|
|
2929
2974
|
}))
|
|
2930
2975
|
}
|
|
2976
|
+
|
|
2977
|
+
return includeLogs ? status : withoutStatusProcessLogs(status)
|
|
2931
2978
|
}
|
|
2932
2979
|
}
|
|
2933
2980
|
|
|
@@ -2939,6 +2986,50 @@ export function ownerConfigDigest(config) {
|
|
|
2939
2986
|
return crypto.createHash("sha256").update(JSON.stringify(config)).digest("hex")
|
|
2940
2987
|
}
|
|
2941
2988
|
|
|
2989
|
+
/**
|
|
2990
|
+
* Removes captured output from a live status response without changing any other status field.
|
|
2991
|
+
* @param {DaemonStatus} status - Full status payload.
|
|
2992
|
+
* @returns {DaemonStatusWithoutLogs} Log-free status payload.
|
|
2993
|
+
*/
|
|
2994
|
+
function withoutStatusProcessLogs(status) {
|
|
2995
|
+
return {
|
|
2996
|
+
...status,
|
|
2997
|
+
releases: status.releases.map((release) => ({
|
|
2998
|
+
...release,
|
|
2999
|
+
processes: release.processes.map(withoutProcessLogs)
|
|
3000
|
+
})),
|
|
3001
|
+
services: status.services.map(({process, ...service}) => ({
|
|
3002
|
+
...service,
|
|
3003
|
+
process: withoutProcessLogs(process)
|
|
3004
|
+
})),
|
|
3005
|
+
singletons: status.singletons.map(({process, ...singleton}) => ({
|
|
3006
|
+
...singleton,
|
|
3007
|
+
process: withoutProcessLogs(process)
|
|
3008
|
+
}))
|
|
3009
|
+
}
|
|
3010
|
+
}
|
|
3011
|
+
|
|
3012
|
+
/**
|
|
3013
|
+
* @param {import("./managed-process.js").ManagedProcessStatus} processStatus - Full managed-process status.
|
|
3014
|
+
* @returns {ManagedProcessStatusWithoutLogs} Status without captured process output.
|
|
3015
|
+
*/
|
|
3016
|
+
function withoutProcessLogs(processStatus) {
|
|
3017
|
+
const {logs: _logs, ...withoutLogs} = processStatus
|
|
3018
|
+
|
|
3019
|
+
return withoutLogs
|
|
3020
|
+
}
|
|
3021
|
+
|
|
3022
|
+
/**
|
|
3023
|
+
* @param {JsonValue} value - Optional status request field.
|
|
3024
|
+
* @returns {boolean} Whether captured process output is included.
|
|
3025
|
+
*/
|
|
3026
|
+
function statusIncludeLogs(value) {
|
|
3027
|
+
if (value === undefined) return true
|
|
3028
|
+
if (typeof value !== "boolean") throw new Error("includeLogs must be a boolean")
|
|
3029
|
+
|
|
3030
|
+
return value
|
|
3031
|
+
}
|
|
3032
|
+
|
|
2942
3033
|
/**
|
|
2943
3034
|
* @param {PrivateOwnerState} transfer - Authenticated private incumbent state.
|
|
2944
3035
|
* @param {GenerationTransition} transition - Unresolved generation transition.
|
|
@@ -2966,6 +3057,7 @@ function terminalRetirementFailure(failure) {
|
|
|
2966
3057
|
return typeof failure === "string" && (
|
|
2967
3058
|
/^Cannot activate .+ generation from retired$/iu.test(failure) ||
|
|
2968
3059
|
/^activate command exited non-zero with status \d+$/u.test(failure) ||
|
|
3060
|
+
/^Process .+ is not retained for reactivation$/u.test(failure) ||
|
|
2969
3061
|
/^Release .+ retirement quiescence failed: quiet command exited non-zero with status \d+$/u.test(failure)
|
|
2970
3062
|
)
|
|
2971
3063
|
}
|
package/src/process-guardian.js
CHANGED
|
@@ -631,7 +631,11 @@ async function execute(request, socket) {
|
|
|
631
631
|
},
|
|
632
632
|
shouldRestart: () => record.desired
|
|
633
633
|
}
|
|
634
|
-
|
|
634
|
+
// Only process keys inventoried from the legacy guardian remain delegated to it.
|
|
635
|
+
// New release processes must be owned by the current guardian so current lifecycle
|
|
636
|
+
// fields (including activation timeouts) are enforced instead of downgraded by an
|
|
637
|
+
// older guardian protocol implementation.
|
|
638
|
+
const managedProcess = recoversLegacyProcess && legacyGuardian
|
|
635
639
|
? legacyGuardian.process(request.key, managedDefinition)
|
|
636
640
|
: new ManagedProcess(managedDefinition)
|
|
637
641
|
|
package/src/release-group.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
|
+
import {spawn} from "node:child_process"
|
|
3
4
|
import {EventEmitter} from "node:events"
|
|
4
5
|
import ManagedProcess from "./managed-process.js"
|
|
5
6
|
import {findAvailablePort} from "./port-allocator.js"
|
|
@@ -14,6 +15,40 @@ import {waitForHealth} from "./health.js"
|
|
|
14
15
|
* @typedef {{count?: number, guardianKey?: string, index?: number, instanceId?: string, shouldRestart?: () => boolean}} BuildProcessOptions
|
|
15
16
|
*/
|
|
16
17
|
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Runs a lifecycle authority command in the current daemon so nested legacy guardians
|
|
21
|
+
* cannot silently downgrade lifecycle timeout semantics for newly deployed processes.
|
|
22
|
+
* @param {{command: string, cwd: string, env: Record<string, string>, label: string, timeoutMs: number}} options - Command contract.
|
|
23
|
+
* @returns {Promise<void>} Completion after an acknowledged command.
|
|
24
|
+
*/
|
|
25
|
+
async function runLifecycleCommand({command, cwd, env, label, timeoutMs}) {
|
|
26
|
+
await /** @type {Promise<void>} */ (new Promise((resolve, reject) => {
|
|
27
|
+
const child = spawn(command, {cwd, detached: true, env: {...process.env, ...env}, shell: true, stdio: "ignore"})
|
|
28
|
+
let settled = false
|
|
29
|
+
/** @param {Error | undefined} error - Completion failure. */
|
|
30
|
+
/** @param {Error | undefined} error - Completion failure. */
|
|
31
|
+
const finish = (error) => {
|
|
32
|
+
if (settled) return
|
|
33
|
+
settled = true
|
|
34
|
+
clearTimeout(timer)
|
|
35
|
+
if (error) reject(error)
|
|
36
|
+
else resolve()
|
|
37
|
+
}
|
|
38
|
+
const timer = setTimeout(() => {
|
|
39
|
+
if (child.pid) {
|
|
40
|
+
try { process.kill(-child.pid, "SIGKILL") } catch { /* The command already exited. */ }
|
|
41
|
+
}
|
|
42
|
+
finish(new Error(`${label} timed out after ${timeoutMs}ms`))
|
|
43
|
+
}, timeoutMs)
|
|
44
|
+
child.once("error", finish)
|
|
45
|
+
child.once("exit", (code, signal) => {
|
|
46
|
+
if (code === 0) finish(undefined)
|
|
47
|
+
else finish(new Error(`${label} exited non-zero with ${signal ? `signal ${signal}` : `status ${code ?? "unknown"}`}`))
|
|
48
|
+
})
|
|
49
|
+
}))
|
|
50
|
+
}
|
|
51
|
+
|
|
17
52
|
/**
|
|
18
53
|
* @param {string} id - Process id.
|
|
19
54
|
* @returns {string} Environment suffix.
|
|
@@ -276,7 +311,19 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
276
311
|
const [instance] = this.getProcesses(processConfig.id)
|
|
277
312
|
|
|
278
313
|
if (!instance) throw new Error(`Generation activation process ${processConfig.id} is not running for release ${this.releaseId}`)
|
|
279
|
-
|
|
314
|
+
if (instance.process.constructor.name === ManagedProcess.name) {
|
|
315
|
+
await instance.process.activateStrict()
|
|
316
|
+
} else {
|
|
317
|
+
const definition = this.processDefinition(processConfig)
|
|
318
|
+
await runLifecycleCommand({
|
|
319
|
+
command: /** @type {string} */ (processConfig.lifecycle.activateCommand),
|
|
320
|
+
cwd: /** @type {string} */ (definition.cwd),
|
|
321
|
+
env: /** @type {Record<string, string>} */ (definition.env),
|
|
322
|
+
label: "activate command",
|
|
323
|
+
timeoutMs: processConfig.lifecycle.activateTimeoutMs ?? 30000
|
|
324
|
+
})
|
|
325
|
+
await instance.process.setLifecycleRole("active")
|
|
326
|
+
}
|
|
280
327
|
}
|
|
281
328
|
|
|
282
329
|
/** Restores the retained generation coordinator to its active role in place. */
|
|
@@ -6,6 +6,7 @@ import net from "node:net"
|
|
|
6
6
|
import os from "node:os"
|
|
7
7
|
import path from "node:path"
|
|
8
8
|
import test, {after, before} from "node:test"
|
|
9
|
+
import {fileURLToPath} from "node:url"
|
|
9
10
|
import RollbridgeDaemon from "../src/daemon.js"
|
|
10
11
|
import {normalizeConfig} from "../src/config.js"
|
|
11
12
|
import {sendControlCommand} from "../src/control-client.js"
|
|
@@ -13,6 +14,9 @@ import {sendControlCommand} from "../src/control-client.js"
|
|
|
13
14
|
let root = ""
|
|
14
15
|
let socketPath = ""
|
|
15
16
|
let daemon = /** @type {RollbridgeDaemon | undefined} */ (undefined)
|
|
17
|
+
const currentDir = path.dirname(fileURLToPath(import.meta.url))
|
|
18
|
+
const dummyAppPath = path.join(currentDir, "fixtures", "dummy-app.js")
|
|
19
|
+
const runningProcessCommand = `${JSON.stringify(process.execPath)} -e ${JSON.stringify("setInterval(() => {}, 1000)")}`
|
|
16
20
|
|
|
17
21
|
before(async () => {
|
|
18
22
|
root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-control-"))
|
|
@@ -21,7 +25,11 @@ before(async () => {
|
|
|
21
25
|
const config = normalizeConfig({
|
|
22
26
|
application: "rollbridge-control-test",
|
|
23
27
|
control: {path: socketPath},
|
|
24
|
-
processes: [
|
|
28
|
+
processes: [
|
|
29
|
+
{command: runningProcessCommand, id: "beacon", policy: "service", port: {from: 0, to: 0}},
|
|
30
|
+
{command: `${JSON.stringify(process.execPath)} ${JSON.stringify(dummyAppPath)}`, health: {intervalMs: 50, path: "/ping", timeoutMs: 3000}, id: "web", policy: "proxied", port: {from: 0, to: 0}},
|
|
31
|
+
{command: runningProcessCommand, id: "jobs-main", policy: "singleton"}
|
|
32
|
+
],
|
|
25
33
|
proxy: {host: "127.0.0.1", port: 0}
|
|
26
34
|
})
|
|
27
35
|
|
|
@@ -92,3 +100,55 @@ test("a known command missing a required field returns a field error", async ()
|
|
|
92
100
|
assert.equal(response.status, "error")
|
|
93
101
|
assert.equal(response.error, "releasePath is required")
|
|
94
102
|
})
|
|
103
|
+
|
|
104
|
+
test("status can omit process logs without changing the default status payload", async () => {
|
|
105
|
+
assert.ok(daemon)
|
|
106
|
+
await daemon.deploy({releaseId: "v1", releasePath: root, revision: "v1"})
|
|
107
|
+
|
|
108
|
+
const full = /** @type {import("../src/daemon.js").DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: socketPath}))
|
|
109
|
+
const compact = /** @type {import("../src/daemon.js").DaemonStatusWithoutLogs} */ (await sendControlCommand({command: {command: "status", includeLogs: false}, path: socketPath}))
|
|
110
|
+
const invalid = await sendRawControlLine(JSON.stringify({command: "status", includeLogs: "false"}))
|
|
111
|
+
|
|
112
|
+
assert.ok(full.releases[0]?.processes.every((processStatus) => Array.isArray(processStatus.logs)))
|
|
113
|
+
assert.ok(full.services.every(({process: processStatus}) => Array.isArray(processStatus.logs)))
|
|
114
|
+
assert.ok(full.singletons.every(({process: processStatus}) => Array.isArray(processStatus.logs)))
|
|
115
|
+
assert.deepEqual(statusWithoutProcessUptimes(compact), statusWithoutProcessUptimes({
|
|
116
|
+
...full,
|
|
117
|
+
releases: full.releases.map((release) => ({
|
|
118
|
+
...release,
|
|
119
|
+
processes: release.processes.map(({logs: _logs, ...processStatus}) => processStatus)
|
|
120
|
+
})),
|
|
121
|
+
services: full.services.map(({process, ...service}) => ({
|
|
122
|
+
...service,
|
|
123
|
+
process: (({logs: _logs, ...processStatus}) => processStatus)(process)
|
|
124
|
+
})),
|
|
125
|
+
singletons: full.singletons.map(({process, ...singleton}) => ({
|
|
126
|
+
...singleton,
|
|
127
|
+
process: (({logs: _logs, ...processStatus}) => processStatus)(process)
|
|
128
|
+
}))
|
|
129
|
+
}))
|
|
130
|
+
assert.equal(invalid.status, "error")
|
|
131
|
+
assert.equal(invalid.error, "includeLogs must be a boolean")
|
|
132
|
+
})
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* @param {import("../src/daemon.js").DaemonStatus | import("../src/daemon.js").DaemonStatusWithoutLogs} status - Status response.
|
|
136
|
+
* @returns {Record<string, import("../src/json.js").JsonValue>} Status without volatile process uptime.
|
|
137
|
+
*/
|
|
138
|
+
function statusWithoutProcessUptimes(status) {
|
|
139
|
+
return {
|
|
140
|
+
...status,
|
|
141
|
+
releases: status.releases.map((release) => ({
|
|
142
|
+
...release,
|
|
143
|
+
processes: release.processes.map(({uptimeMs: _uptimeMs, ...processStatus}) => processStatus)
|
|
144
|
+
})),
|
|
145
|
+
services: status.services.map(({process, ...service}) => ({
|
|
146
|
+
...service,
|
|
147
|
+
process: (({uptimeMs: _uptimeMs, ...processStatus}) => processStatus)(process)
|
|
148
|
+
})),
|
|
149
|
+
singletons: status.singletons.map(({process, ...singleton}) => ({
|
|
150
|
+
...singleton,
|
|
151
|
+
process: (({uptimeMs: _uptimeMs, ...processStatus}) => processStatus)(process)
|
|
152
|
+
}))
|
|
153
|
+
}
|
|
154
|
+
}
|
|
@@ -70,6 +70,21 @@ test("guardian runs a strict activation lifecycle command for the exact register
|
|
|
70
70
|
}
|
|
71
71
|
})
|
|
72
72
|
|
|
73
|
+
test("guardian preserves a custom activation timeout in the registered process definition", async () => {
|
|
74
|
+
const fixture = await createGuardian()
|
|
75
|
+
const processInstance = fixture.client.process("candidate-activation-timeout", {
|
|
76
|
+
...definition("candidate-activation-timeout"),
|
|
77
|
+
lifecycle: {activateCommand: "sleep 0.05", activateTimeoutMs: 10, drainTimeoutMs: 0}
|
|
78
|
+
})
|
|
79
|
+
|
|
80
|
+
try {
|
|
81
|
+
await processInstance.start()
|
|
82
|
+
await assert.rejects(() => processInstance.activateStrict(), /activate command timed out after 10ms/i)
|
|
83
|
+
} finally {
|
|
84
|
+
await cleanupGuardian(fixture)
|
|
85
|
+
}
|
|
86
|
+
})
|
|
87
|
+
|
|
73
88
|
test("client reactivates a retained process through a guardian without the reactivation command", async () => {
|
|
74
89
|
const fixture = await createGuardian()
|
|
75
90
|
const lifecyclePath = path.join(fixture.root, "lifecycle.log")
|
|
@@ -1501,6 +1516,14 @@ test("first upgrade migrates a real pre-split guardian without replacing its own
|
|
|
1501
1516
|
assert.deepEqual(await upgraded.stageOwnerReplacement(prepared.replacementId, {authority: nextAuthority, snapshot: ownerState.snapshot}), {committed: true})
|
|
1502
1517
|
await committed
|
|
1503
1518
|
assert.equal(restored.status().pid, legacyPid)
|
|
1519
|
+
|
|
1520
|
+
const currentProcess = upgraded.process("release:v2:current-worker", {
|
|
1521
|
+
...definition("current-worker"),
|
|
1522
|
+
lifecycle: {activateCommand: "sleep 0.05", activateTimeoutMs: 10, drainTimeoutMs: 0}
|
|
1523
|
+
})
|
|
1524
|
+
await currentProcess.start()
|
|
1525
|
+
await assert.rejects(() => currentProcess.activateStrict(), /activate command timed out after 10ms/i)
|
|
1526
|
+
assert.equal(restored.status().pid, legacyPid)
|
|
1504
1527
|
await upgraded.shutdown()
|
|
1505
1528
|
await upgraded.guardianExit()
|
|
1506
1529
|
} finally {
|
package/test/logs.test.js
CHANGED
|
@@ -91,6 +91,13 @@ test("logs CLI prints captured output per managed process", async () => {
|
|
|
91
91
|
assert.ok(web, "expected a web entry in the JSON output")
|
|
92
92
|
assert.match(web.source, /release v1 \(active\)/)
|
|
93
93
|
assert.ok(Array.isArray(web.logs))
|
|
94
|
+
|
|
95
|
+
lines.length = 0
|
|
96
|
+
await runCli(["node", "rollbridge", "status", "--no-logs", "-c", path.join(root, "rollbridge.js")])
|
|
97
|
+
|
|
98
|
+
const status = JSON.parse(lines.join("\n"))
|
|
99
|
+
|
|
100
|
+
assert.equal("logs" in status.releases[0].processes[0], false)
|
|
94
101
|
} finally {
|
|
95
102
|
console.log = originalLog
|
|
96
103
|
await daemon.shutdown()
|
|
@@ -171,7 +171,7 @@ test("exact bootstrap restores the committed generation after external owner ret
|
|
|
171
171
|
assert.ok(active.services.every(({process}) => typeof process.pid === "number" && process.state === "running"))
|
|
172
172
|
assert.ok(active.singletons.every(({process}) => typeof process.pid === "number" && process.state === "running"))
|
|
173
173
|
assert.deepEqual(recoveryOrder, ["service", "activate", "singleton"], "candidate activation must precede post-commit singleton completion")
|
|
174
|
-
assert.deepEqual(await
|
|
174
|
+
assert.deepEqual(await waitForLifecycleEvents(fixture.lifecycleLogPath, ["activate:v1", "retire:v1", "activate:v2", "retire:v2", "activate:v2"]), ["activate:v1", "retire:v1", "activate:v2", "retire:v2", "activate:v2"])
|
|
175
175
|
} finally {
|
|
176
176
|
if (recovered) {
|
|
177
177
|
const activeRecovery = recovered.status().activeReleaseId === "v2"
|
|
@@ -307,7 +307,7 @@ test("journaled committed bootstrap recovery resumes after a restart begins", as
|
|
|
307
307
|
assert.ok(active.singletons.every(({process}) => typeof process.pid === "number" && process.state === "running"))
|
|
308
308
|
assert.equal(releaseProcessPid(active, "v1", "worker"), v1WorkerPid)
|
|
309
309
|
assert.equal(isProcessRunning(v1WorkerPid), true)
|
|
310
|
-
assert.deepEqual(await
|
|
310
|
+
assert.deepEqual(await waitForLifecycleEvents(fixture.lifecycleLogPath, ["activate:v1", "retire:v1", "activate:v2", "retire:v2", "activate:v2"]), ["activate:v1", "retire:v1", "activate:v2", "retire:v2", "activate:v2"])
|
|
311
311
|
} finally {
|
|
312
312
|
if (recovered) {
|
|
313
313
|
const shutdown = recovered.shutdown()
|
|
@@ -873,7 +873,7 @@ test("owner recovery preserves complete private transition authority across a ca
|
|
|
873
873
|
|
|
874
874
|
assert.equal(recovered.status().activeReleaseId, "v2")
|
|
875
875
|
assert.equal(recovered.status().generationTransition?.phase, "committed")
|
|
876
|
-
assert.deepEqual(await
|
|
876
|
+
assert.deepEqual(await waitForLifecycleEvents(fixture.lifecycleLogPath, ["activate:v1", "retire:v1", "activate:v2"]), ["activate:v1", "retire:v1", "activate:v2"])
|
|
877
877
|
} finally {
|
|
878
878
|
if (recovered) await recovered.shutdown().catch(() => {})
|
|
879
879
|
await stopFixtureGuardian(fixture.statePath)
|
|
@@ -988,6 +988,7 @@ test("owner recovery uses the owning release singleton definition during a commi
|
|
|
988
988
|
assert.equal(pending.activeReleaseId, "v2")
|
|
989
989
|
assert.equal(pending.generationTransition?.phase, "committed_pending")
|
|
990
990
|
assert.equal(pending.singletonReleaseIds?.singleton, "v1")
|
|
991
|
+
await owner.persistState({throwOnError: true})
|
|
991
992
|
await owner.retireCommittedOwner(undefined)
|
|
992
993
|
owner.guardian?.disconnect()
|
|
993
994
|
|
|
@@ -1773,6 +1774,33 @@ async function lifecycleEvents(lifecycleLogPath) {
|
|
|
1773
1774
|
return (await fs.readFile(lifecycleLogPath, "utf8")).trim().split("\n").filter(Boolean)
|
|
1774
1775
|
}
|
|
1775
1776
|
|
|
1777
|
+
/**
|
|
1778
|
+
* @param {string} lifecycleLogPath - Fixture lifecycle log.
|
|
1779
|
+
* @param {string[]} expected - Exact lifecycle events to await.
|
|
1780
|
+
* @returns {Promise<string[]>} The matching ordered events.
|
|
1781
|
+
*/
|
|
1782
|
+
async function waitForLifecycleEvents(lifecycleLogPath, expected) {
|
|
1783
|
+
const watcher = fs.watch(lifecycleLogPath, {signal: AbortSignal.timeout(3000)})
|
|
1784
|
+
|
|
1785
|
+
try {
|
|
1786
|
+
const events = await lifecycleEvents(lifecycleLogPath)
|
|
1787
|
+
|
|
1788
|
+
if (JSON.stringify(events) === JSON.stringify(expected)) return events
|
|
1789
|
+
for await (const _event of watcher) {
|
|
1790
|
+
const changedEvents = await lifecycleEvents(lifecycleLogPath)
|
|
1791
|
+
|
|
1792
|
+
if (JSON.stringify(changedEvents) === JSON.stringify(expected)) return changedEvents
|
|
1793
|
+
}
|
|
1794
|
+
} catch (error) {
|
|
1795
|
+
if (error instanceof Error && error.name === "AbortError") throw new Error(`Timed out waiting for lifecycle events: ${JSON.stringify(expected)}`, {cause: error})
|
|
1796
|
+
throw error
|
|
1797
|
+
} finally {
|
|
1798
|
+
await watcher.return?.()
|
|
1799
|
+
}
|
|
1800
|
+
|
|
1801
|
+
throw new Error(`Timed out waiting for lifecycle events: ${JSON.stringify(expected)}`)
|
|
1802
|
+
}
|
|
1803
|
+
|
|
1776
1804
|
/**
|
|
1777
1805
|
* @param {string} root - Fixture root.
|
|
1778
1806
|
* @param {string} releaseId - Release id.
|
|
@@ -1826,6 +1826,7 @@ test("owner replacement preserves accepted degraded incumbent web authority", as
|
|
|
1826
1826
|
releasePath: v2Path,
|
|
1827
1827
|
revision: "v2"
|
|
1828
1828
|
}, path: socketPath})
|
|
1829
|
+
await waitForReleaseState(socketPath, "v2", "stopped")
|
|
1829
1830
|
const accepted = await sendControlCommand({command: {command: "status"}, path: socketPath})
|
|
1830
1831
|
|
|
1831
1832
|
assert.equal(recovery.jobsStatus, "degraded")
|
|
@@ -1833,7 +1834,6 @@ test("owner replacement preserves accepted degraded incumbent web authority", as
|
|
|
1833
1834
|
assert.equal(/** @type {{phase?: string}} */ (accepted.generationTransition).phase, "degraded_active")
|
|
1834
1835
|
replacement = spawn(process.execPath, [binPath, "daemon", "--config", configPath, "--replace-owner"], {stdio: ["ignore", "pipe", "pipe"]})
|
|
1835
1836
|
await waitForLog(replacement, "owner replacement committed")
|
|
1836
|
-
await new Promise((resolve) => setTimeout(resolve, 250))
|
|
1837
1837
|
const recovered = await sendControlCommand({command: {command: "status"}, path: socketPath})
|
|
1838
1838
|
const response = await fetch(`http://127.0.0.1:${proxyPort}/release`)
|
|
1839
1839
|
|
package/test/rollbridge.test.js
CHANGED
|
@@ -505,7 +505,7 @@ test("candidate activation retires jobs-main with its workers without waiting fo
|
|
|
505
505
|
}
|
|
506
506
|
})
|
|
507
507
|
|
|
508
|
-
test("opt-in generation lifecycle
|
|
508
|
+
test("opt-in generation lifecycle acknowledges old retirement before activating the candidate", async () => {
|
|
509
509
|
const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, nonBlockingDrainWorker: true, webDependsOnService: true})
|
|
510
510
|
const daemon = await startDaemon(fixture.config)
|
|
511
511
|
|
|
@@ -633,7 +633,7 @@ test("first generation is not committed when its activation acknowledgement fail
|
|
|
633
633
|
}
|
|
634
634
|
})
|
|
635
635
|
|
|
636
|
-
test("retirement failure retains the exact transition, blocks other deploys, and exact resume continues it", async () => {
|
|
636
|
+
test("retirement acknowledgement failure retains the exact transition, blocks other deploys, and exact resume continues it", async () => {
|
|
637
637
|
const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, handoffServiceQuietFailure: true, nonBlockingDrainWorker: true, webDependsOnService: true})
|
|
638
638
|
const daemon = await startDaemon(fixture.config)
|
|
639
639
|
|
|
@@ -806,6 +806,8 @@ test("explicit recovery stops the exact failed candidate and fences degraded inc
|
|
|
806
806
|
transition.compensationError = "incumbent activation was temporarily unavailable"
|
|
807
807
|
await assert.rejects(() => exactRecovery(), /terminal retirement/i)
|
|
808
808
|
transition.compensationError = terminalFailure
|
|
809
|
+
await incumbentCoordinator.setLifecycleRole("retired")
|
|
810
|
+
assert.equal(incumbentCoordinator.status().lifecycleRole, "retired")
|
|
809
811
|
const checkpoint = daemon.checkpointGenerationTransition.bind(daemon)
|
|
810
812
|
|
|
811
813
|
daemon.checkpointGenerationTransition = async () => { throw new Error("injected checkpoint failure") }
|
|
@@ -821,7 +823,7 @@ test("explicit recovery stops the exact failed candidate and fences degraded inc
|
|
|
821
823
|
assert.equal(daemon.status().generationTransition?.phase, "degraded_active")
|
|
822
824
|
assert.equal(statusRelease(daemon, "v1").processes.find(({id}) => id === "web")?.pid, incumbentWebPid)
|
|
823
825
|
assert.equal(await fetchText(daemon, "/release"), "v1")
|
|
824
|
-
assert.
|
|
826
|
+
assert.ok(["draining", "stopped"].includes(candidate.state), "guarded recovery returns before failed-candidate drain completion")
|
|
825
827
|
assert.deepEqual(await lifecycleEvents(fixture.lifecycleLogPath), eventsBeforeRecovery, "recovery must not activate either retained generation")
|
|
826
828
|
const persisted = /** @type {{generationTransition?: import("../src/json.js").JsonValue} | undefined} */ (await readState(fixture.statePath))
|
|
827
829
|
|
|
@@ -830,11 +832,21 @@ test("explicit recovery stops the exact failed candidate and fences degraded inc
|
|
|
830
832
|
transition.phase = "retiring_previous"
|
|
831
833
|
transition.error = "Release v1 retirement quiescence failed: quiet command exited non-zero with status 1"
|
|
832
834
|
transition.compensationError = undefined
|
|
835
|
+
candidate.state = "starting"
|
|
833
836
|
await daemon.checkpointGenerationTransition()
|
|
834
837
|
const legacyRecovery = await exactRecovery()
|
|
835
838
|
|
|
836
839
|
assert.equal(legacyRecovery.recoveryStatus, "retired_incumbent_accepted")
|
|
837
840
|
assert.equal(daemon.status().generationTransition?.phase, "degraded_active", "guarded recovery migrates a legacy terminal retirement fence")
|
|
841
|
+
|
|
842
|
+
transition.phase = "restoring_previous"
|
|
843
|
+
transition.compensationError = "Process background-jobs-main is not retained for reactivation"
|
|
844
|
+
daemon.releases.get("v1")?.processes.delete("beacon")
|
|
845
|
+
await daemon.checkpointGenerationTransition()
|
|
846
|
+
const absentCoordinatorRecovery = await exactRecovery()
|
|
847
|
+
|
|
848
|
+
assert.equal(absentCoordinatorRecovery.jobsStatus, "degraded")
|
|
849
|
+
assert.equal(daemon.status().generationTransition?.phase, "degraded_active", "terminally absent incumbent coordinator remains guarded jobs-degraded authority")
|
|
838
850
|
await assert.rejects(() => daemon.deploy({releaseId: "bad-v3", releasePath: fixture.root, revision: "bad-v3"}), /health check failed/i)
|
|
839
851
|
assert.equal(daemon.status().generationTransition?.phase, "degraded_active")
|
|
840
852
|
assert.equal(statusRelease(daemon, "v1").processes.find(({id}) => id === "web")?.pid, incumbentWebPid)
|
|
@@ -1094,6 +1106,7 @@ test("retired generation coordinator remains fenced after exit", async () => {
|
|
|
1094
1106
|
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
1095
1107
|
socket = await openWebSocket(daemon)
|
|
1096
1108
|
await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
1109
|
+
await waitFor(() => statusRelease(daemon, "v1").processes.find((entry) => entry.id === "beacon")?.lifecycleRole === "retired")
|
|
1097
1110
|
const coordinator = statusRelease(daemon, "v1").processes.find((entry) => entry.id === "beacon")
|
|
1098
1111
|
const coordinatorProcess = daemon.releases.get("v1")?.getProcess("beacon")
|
|
1099
1112
|
|