rollbridge 0.1.74 → 0.1.76
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.d/20261118-bounded-control-client-close.md +4 -0
- package/package.json +1 -1
- package/src/cli.js +1 -1
- package/src/control-client.js +32 -5
- package/src/daemon.js +45 -6
- package/src/guardian-client.js +40 -2
- package/src/managed-process.js +31 -3
- package/src/process-guardian.js +15 -0
- package/src/process-memory.js +25 -0
- package/src/release-group.js +33 -13
- package/test/control-protocol.test.js +22 -0
- package/test/guardian-client.test.js +30 -0
- package/test/managed-process.test.js +23 -0
- package/test/owner-recovery.test.js +28 -1
- package/test/owner-replacement.test.js +1 -1
- package/test/release-group.test.js +39 -0
package/package.json
CHANGED
package/src/cli.js
CHANGED
|
@@ -82,7 +82,7 @@ export async function runCli(argv) {
|
|
|
82
82
|
const shutdown = async () => {
|
|
83
83
|
shutdownRequested = true
|
|
84
84
|
await startupPromise.catch(() => undefined)
|
|
85
|
-
await daemon.shutdown()
|
|
85
|
+
await daemon.shutdown({preserveActiveRelease: Boolean(config.ownerRecovery)})
|
|
86
86
|
process.exit(0)
|
|
87
87
|
}
|
|
88
88
|
|
package/src/control-client.js
CHANGED
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
import net from "node:net"
|
|
4
4
|
|
|
5
|
+
const DEFAULT_CONTROL_COMMAND_TIMEOUT_MS = 30000
|
|
6
|
+
|
|
5
7
|
/**
|
|
6
8
|
* @typedef {import("./json.js").JsonValue} JsonValue
|
|
7
9
|
*/
|
|
@@ -11,15 +13,40 @@ import net from "node:net"
|
|
|
11
13
|
* @param {object} args - Options.
|
|
12
14
|
* @param {Record<string, JsonValue>} args.command - Command payload.
|
|
13
15
|
* @param {string} args.path - Control socket path.
|
|
16
|
+
* @param {number} [args.timeoutMs] - Maximum time to wait for a response.
|
|
14
17
|
* @returns {Promise<Record<string, JsonValue>>} Response payload.
|
|
15
18
|
*/
|
|
16
|
-
export async function sendControlCommand({command, path}) {
|
|
19
|
+
export async function sendControlCommand({command, path, timeoutMs = DEFAULT_CONTROL_COMMAND_TIMEOUT_MS}) {
|
|
17
20
|
return await new Promise((resolve, reject) => {
|
|
18
21
|
const socket = net.createConnection(path)
|
|
19
22
|
let buffer = ""
|
|
23
|
+
let settled = false
|
|
24
|
+
const timeout = setTimeout(() => {
|
|
25
|
+
socket.destroy()
|
|
26
|
+
settleReject(new Error(`Rollbridge control command ${String(command.command)} on ${path} timed out after ${timeoutMs}ms`))
|
|
27
|
+
}, timeoutMs)
|
|
28
|
+
|
|
29
|
+
/** @param {Error} error - Command failure. */
|
|
30
|
+
const settleReject = (error) => {
|
|
31
|
+
if (settled) return
|
|
32
|
+
settled = true
|
|
33
|
+
clearTimeout(timeout)
|
|
34
|
+
reject(error)
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** @param {Record<string, JsonValue>} response - Command response. */
|
|
38
|
+
const settleResolve = (response) => {
|
|
39
|
+
if (settled) return
|
|
40
|
+
settled = true
|
|
41
|
+
clearTimeout(timeout)
|
|
42
|
+
resolve(response)
|
|
43
|
+
}
|
|
20
44
|
|
|
21
45
|
socket.setEncoding("utf8")
|
|
22
|
-
socket.once("error",
|
|
46
|
+
socket.once("error", settleReject)
|
|
47
|
+
socket.once("close", () => {
|
|
48
|
+
if (!settled) settleReject(new Error(`Rollbridge control connection ${path} closed before its response`))
|
|
49
|
+
})
|
|
23
50
|
socket.on("data", (chunk) => {
|
|
24
51
|
buffer += chunk
|
|
25
52
|
const newlineIndex = buffer.indexOf("\n")
|
|
@@ -33,16 +60,16 @@ export async function sendControlCommand({command, path}) {
|
|
|
33
60
|
response = JSON.parse(line)
|
|
34
61
|
} catch (error) {
|
|
35
62
|
socket.destroy()
|
|
36
|
-
|
|
63
|
+
settleReject(new Error(`Invalid Rollbridge control response from ${path}`, {cause: error}))
|
|
37
64
|
return
|
|
38
65
|
}
|
|
39
66
|
|
|
40
67
|
socket.end()
|
|
41
68
|
|
|
42
69
|
if (response.status === "error") {
|
|
43
|
-
|
|
70
|
+
settleReject(new Error(String(response.error || "Unknown Rollbridge error")))
|
|
44
71
|
} else {
|
|
45
|
-
|
|
72
|
+
settleResolve(response)
|
|
46
73
|
}
|
|
47
74
|
})
|
|
48
75
|
socket.write(`${JSON.stringify(command)}\n`)
|
package/src/daemon.js
CHANGED
|
@@ -25,7 +25,7 @@ const STATE_PERSIST_INTERVAL_MS = 5000
|
|
|
25
25
|
* @typedef {{releaseId?: string, releasePath: string, revision?: string}} DeployArgs
|
|
26
26
|
* @typedef {{attestation?: string, releaseId: string, releasePath: string, revision: string}} BootstrapIdentity
|
|
27
27
|
* @typedef {{id: string, process: import("./managed-process.js").ManagedProcessStatus}} ProcessStatus
|
|
28
|
-
* @typedef {{disruptive:
|
|
28
|
+
* @typedef {{disruptive: boolean, mode: "guardian-version-handoff" | "legacy-first-upgrade", reason: string}} OwnerTransition
|
|
29
29
|
* @typedef {"guardian-version-drift" | "legacy-protocol"} GuardianHandoffProtocol
|
|
30
30
|
* @typedef {"candidate_ready" | "retiring_previous" | "previous_retired" | "activating_candidate" | "restoring_previous" | "retiring_failed_candidate" | "degraded_active" | "committed_pending" | "committed" | "restoring_committed"} GenerationTransitionPhase
|
|
31
31
|
* @typedef {{activationError?: string, activationLifecycle?: boolean, candidateReleaseId: string, candidateReleasePath: string, candidateRevision: string, compensationError?: string, configDigest: string, degradedIncumbent?: boolean, error?: string, journalRevision?: number, phase: GenerationTransitionPhase, previousReleaseId: string | null, startedAt: string, updatedAt: string}} GenerationTransition
|
|
@@ -412,7 +412,11 @@ export default class RollbridgeDaemon {
|
|
|
412
412
|
shouldStart: () => !this.stopping
|
|
413
413
|
})
|
|
414
414
|
|
|
415
|
-
await release.restore(releaseStatus, {
|
|
415
|
+
await release.restore(releaseStatus, {
|
|
416
|
+
adoptCommittedGeneration: releaseStatus.releaseId === snapshot.activeReleaseId && snapshot.generationTransition?.phase === "committed" && snapshot.ownerTransition?.mode === "guardian-version-handoff",
|
|
417
|
+
configDigest: ownerConfigDigest(config),
|
|
418
|
+
synchronizeLifecycleRole: synchronizeLifecycleRoles
|
|
419
|
+
})
|
|
416
420
|
this.releases.set(release.releaseId, release)
|
|
417
421
|
if (release.releaseId === snapshot.activeReleaseId) this.activeRelease = release
|
|
418
422
|
}
|
|
@@ -3051,17 +3055,52 @@ export default class RollbridgeDaemon {
|
|
|
3051
3055
|
|
|
3052
3056
|
/**
|
|
3053
3057
|
* Stops proxy, control socket, and child processes.
|
|
3054
|
-
* @param {{completionSocket?: net.Socket, waitForControlConnections?: boolean}} [options] - Shutdown connection behavior.
|
|
3058
|
+
* @param {{completionSocket?: net.Socket, preserveActiveRelease?: boolean, waitForControlConnections?: boolean}} [options] - Shutdown connection behavior.
|
|
3055
3059
|
* @returns {Promise<void>} Resolves when owned resources are stopped (and, by default, control connections close).
|
|
3056
3060
|
*/
|
|
3057
|
-
async shutdown({completionSocket, waitForControlConnections = true} = {}) {
|
|
3058
|
-
|
|
3059
|
-
|
|
3061
|
+
async shutdown({completionSocket, preserveActiveRelease = false, waitForControlConnections = true} = {}) {
|
|
3062
|
+
// A supervisor signal during committed_pending is after candidate activation: the candidate
|
|
3063
|
+
// is already serving, so the recovery shutdown may finish the bounded retirement. Earlier
|
|
3064
|
+
// transition phases still refuse shutdown rather than risking the active incumbent.
|
|
3065
|
+
const finishCommittedPendingRetirement = preserveActiveRelease && this.generationTransition?.phase === "committed_pending"
|
|
3066
|
+
|
|
3067
|
+
if (!finishCommittedPendingRetirement) this.assertNoGenerationTransitionRecovery("shut down")
|
|
3068
|
+
if (!this.shutdownPromise) {
|
|
3069
|
+
this.shutdownPromise = finishCommittedPendingRetirement
|
|
3070
|
+
? this.performShutdown(completionSocket)
|
|
3071
|
+
: preserveActiveRelease && this.guardian
|
|
3072
|
+
? this.performRecoveryShutdown(completionSocket)
|
|
3073
|
+
: this.performShutdown(completionSocket)
|
|
3074
|
+
}
|
|
3060
3075
|
|
|
3061
3076
|
await this.shutdownPromise
|
|
3062
3077
|
if (waitForControlConnections && this.controlClosePromise) await this.controlClosePromise
|
|
3063
3078
|
}
|
|
3064
3079
|
|
|
3080
|
+
/**
|
|
3081
|
+
* Relinquishes this daemon during supervisor recovery without stopping guardian-owned
|
|
3082
|
+
* generations. The guardian will start or reconnect the replacement owner, which is the only
|
|
3083
|
+
* point at which the active release may be retired.
|
|
3084
|
+
* @param {net.Socket | undefined} completionSocket - Requesting connection, if any.
|
|
3085
|
+
* @returns {Promise<void>} Resolves after this daemon's listeners are released.
|
|
3086
|
+
*/
|
|
3087
|
+
async performRecoveryShutdown(completionSocket) {
|
|
3088
|
+
this.ownerRetired = true
|
|
3089
|
+
this.controlCommandsReady = false
|
|
3090
|
+
this.stopping = true
|
|
3091
|
+
if (this.persistTimer) clearInterval(this.persistTimer)
|
|
3092
|
+
this.persistTimer = undefined
|
|
3093
|
+
if (this.pendingWrite) await this.pendingWrite
|
|
3094
|
+
this.persistenceEnabled = false
|
|
3095
|
+
this.stateCleanupEnabled = false
|
|
3096
|
+
this.controlClosePromise = this.closeServer(this.controlServer)
|
|
3097
|
+
this.proxyClosePromise = this.closeServer(this.proxyServer)
|
|
3098
|
+
for (const socket of this.controlSockets) if (socket !== completionSocket) socket.destroy()
|
|
3099
|
+
await this.removeControlSocket()
|
|
3100
|
+
await Promise.all([this.controlClosePromise, this.proxyClosePromise])
|
|
3101
|
+
this.guardian?.disconnect()
|
|
3102
|
+
}
|
|
3103
|
+
|
|
3065
3104
|
/**
|
|
3066
3105
|
* Relinquishes stable listeners promptly while retaining draining children under
|
|
3067
3106
|
* this daemon until their normal stop contract completes.
|
package/src/guardian-client.js
CHANGED
|
@@ -516,18 +516,56 @@ class GuardianProcess extends ManagedProcess {
|
|
|
516
516
|
if (!this.registration) {
|
|
517
517
|
this.registration = this.client.request({command: "register", definition: this.definition, key: this.key, provenance: this.provenance})
|
|
518
518
|
.then((status) => { this.cachedStatus = asProcessStatus(status) })
|
|
519
|
+
.catch(async (error) => {
|
|
520
|
+
// Guardians predating stale-record re-registration reject a restore before
|
|
521
|
+
// the current guardian can apply its stopped/failed-record fence. Reconcile
|
|
522
|
+
// only an exact, conclusively dead record through the authenticated remove
|
|
523
|
+
// command; live or ambiguous records remain fail-closed on provenance.
|
|
524
|
+
if (!(error instanceof Error) || !error.message.startsWith(`Guardian provenance mismatch for ${this.key}`)) throw error
|
|
525
|
+
const stale = (await this.client.inventory()).find((entry) => entry.key === this.key)
|
|
526
|
+
const staleStatus = stale?.status
|
|
527
|
+
const staleIsGone = staleStatus &&
|
|
528
|
+
(staleStatus.state === "stopped" || staleStatus.state === "failed") &&
|
|
529
|
+
staleStatus.pid == null &&
|
|
530
|
+
(!staleStatus.heldProcessGroup || staleStatus.heldProcessGroup.liveMemberCount === 0)
|
|
531
|
+
|
|
532
|
+
if (!stale || !staleIsGone) throw error
|
|
533
|
+
await this.client.remove(stale.key, stale.provenance)
|
|
534
|
+
const status = await this.client.request({command: "register", definition: this.definition, key: this.key, provenance: this.provenance})
|
|
535
|
+
|
|
536
|
+
this.cachedStatus = asProcessStatus(status)
|
|
537
|
+
})
|
|
538
|
+
}
|
|
539
|
+
try {
|
|
540
|
+
await this.registration
|
|
541
|
+
} catch (error) {
|
|
542
|
+
this.registration = undefined
|
|
543
|
+
throw error
|
|
519
544
|
}
|
|
520
|
-
await this.registration
|
|
521
545
|
}
|
|
522
546
|
|
|
523
547
|
/** Reconnects to an already registered guardian process without changing its desired state. */
|
|
524
|
-
|
|
548
|
+
/** @param {{adopt?: {configDigest: string, pid: number, processStartTime: string}}} [options] - Optional identity-fenced adoption. */
|
|
549
|
+
async recover({adopt} = {}) {
|
|
525
550
|
if (this.client.reservedProcessKey === this.key) {
|
|
526
551
|
if (this.client.reservedProcessProvenance !== this.provenance) throw new Error(`Guardian provenance mismatch for reserved process ${this.key}`)
|
|
527
552
|
this.cachedStatus = asProcessStatus(await this.client.request({command: "status", key: this.key}))
|
|
553
|
+
if (adopt) await this.adopt(adopt, adopt.configDigest)
|
|
528
554
|
return
|
|
529
555
|
}
|
|
530
556
|
await this.ensureRegistered()
|
|
557
|
+
if (adopt) await this.adopt(adopt, adopt.configDigest)
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
/** Re-adopts a live committed process after a guardian/version handoff. */
|
|
561
|
+
/**
|
|
562
|
+
* @param {{pid: number, processStartTime: string}} identity - Exact process identity.
|
|
563
|
+
* @param {string} [configDigest] - Committed owner config digest.
|
|
564
|
+
*/
|
|
565
|
+
async adopt(identity, configDigest) {
|
|
566
|
+
if (!configDigest) throw new Error("Guardian process adoption requires the committed owner config digest")
|
|
567
|
+
await this.ensureRegistered()
|
|
568
|
+
this.cachedStatus = asProcessStatus(await this.client.request({command: "adopt", key: this.key, pid: identity.pid, processStartTime: identity.processStartTime, configDigest}))
|
|
531
569
|
}
|
|
532
570
|
|
|
533
571
|
/** Attaches a reconstructed process whose incumbent-owned registration was reserved through commit. */
|
package/src/managed-process.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import {EventEmitter} from "node:events"
|
|
4
4
|
import {spawn} from "node:child_process"
|
|
5
|
-
import {processGroupHasLiveMembers, processGroupMembers, sampleProcessGroup} from "./process-memory.js"
|
|
5
|
+
import {processGroupHasLiveMembers, processGroupMembers, processStartTime, sampleProcessGroup} from "./process-memory.js"
|
|
6
6
|
|
|
7
7
|
const DEFAULT_ACTIVATION_HOOK_TIMEOUT_MS = 30000
|
|
8
8
|
const MAX_BUFFERED_OUTPUT_CHARACTERS = 64 * 1024
|
|
@@ -24,7 +24,7 @@ const ESCALATION_KILL_CONFIRM_MS = 5000
|
|
|
24
24
|
* @typedef {import("./config.js").StopTimeoutMs} StopTimeoutMs
|
|
25
25
|
* @typedef {{command: string, cwd: string | undefined, env: Record<string, string | undefined>, lifecycle: import("./config.js").LifecycleConfig, logger: (message: string, data?: Record<string, import("./json.js").JsonValue>) => void, memory: import("./config.js").MemoryConfig | undefined, outputLines: number, restart: import("./config.js").RestartConfig, restartDelayMs: number, shouldRestart: () => boolean, shouldStop?: () => boolean, stopSignal: string, stopTimeoutMs: StopTimeoutMs}} ManagedProcessDefinition
|
|
26
26
|
* @typedef {{liveMemberCount: number | undefined, pgid: number}} HeldProcessGroup
|
|
27
|
-
* @typedef {{children: import("./process-memory.js").ProcessGroupMember[], command: string, cwd: string | undefined, exitCode: number | null | undefined, exitSignal: ProcessExitSignal | undefined, heldProcessGroup: HeldProcessGroup | undefined, id: string, lastMemoryRestartAt: string | undefined, lastStartReason: ManagedProcessStartReason | undefined, lifecycleRole?: LifecycleRole, logs: ManagedProcessLog[], memoryRestarts: number, pid: number | undefined, restarts: number, retirementTimedOut: boolean, rssBytes: number | undefined, startedAt: string | undefined, state: ManagedProcessState, uptimeMs: number | undefined}} ManagedProcessStatus
|
|
27
|
+
* @typedef {{children: import("./process-memory.js").ProcessGroupMember[], command: string, cwd: string | undefined, exitCode: number | null | undefined, exitSignal: ProcessExitSignal | undefined, heldProcessGroup: HeldProcessGroup | undefined, id: string, lastMemoryRestartAt: string | undefined, lastStartReason: ManagedProcessStartReason | undefined, lifecycleRole?: LifecycleRole, logs: ManagedProcessLog[], memoryRestarts: number, pid: number | undefined, processStartTime?: string, restarts: number, retirementTimedOut: boolean, rssBytes: number | undefined, startedAt: string | undefined, state: ManagedProcessState, uptimeMs: number | undefined}} ManagedProcessStatus
|
|
28
28
|
*/
|
|
29
29
|
|
|
30
30
|
export default class ManagedProcess extends EventEmitter {
|
|
@@ -92,6 +92,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
92
92
|
this.child = undefined
|
|
93
93
|
this.exitPromise = undefined
|
|
94
94
|
this.pid = undefined
|
|
95
|
+
this.processStartTime = undefined
|
|
95
96
|
this.exitCode = undefined
|
|
96
97
|
this.exitSignal = undefined
|
|
97
98
|
}
|
|
@@ -144,6 +145,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
144
145
|
child.once("spawn", () => {
|
|
145
146
|
void (async () => {
|
|
146
147
|
this.startedAtMs = Date.now()
|
|
148
|
+
this.processStartTime = child.pid ? processStartTime(child.pid) : undefined
|
|
147
149
|
this.lastStartReason = reason
|
|
148
150
|
const lifecycleRestore = this.restoreLifecycleRole()
|
|
149
151
|
const lifecycleRestoreBarrier = lifecycleRestore.then(() => undefined, () => undefined)
|
|
@@ -179,6 +181,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
179
181
|
if (this.child === child) {
|
|
180
182
|
this.child = undefined
|
|
181
183
|
this.pid = undefined
|
|
184
|
+
this.processStartTime = undefined
|
|
182
185
|
this.exitPromise = undefined
|
|
183
186
|
}
|
|
184
187
|
reject(error)
|
|
@@ -192,6 +195,29 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
192
195
|
})
|
|
193
196
|
}
|
|
194
197
|
|
|
198
|
+
/**
|
|
199
|
+
* Re-adopts an already-live process after a guardian/version handoff.
|
|
200
|
+
* @param {{pid: number, processStartTime: string}} identity - Exact process identity.
|
|
201
|
+
* @param {string} [_configDigest] - Owner digest accepted by guardian-backed implementations.
|
|
202
|
+
* @returns {Promise<void>}
|
|
203
|
+
*/
|
|
204
|
+
async adopt({pid, processStartTime: expectedStartTime}, _configDigest) {
|
|
205
|
+
if (!Number.isInteger(pid) || pid <= 0 || !/^\d+$/.test(expectedStartTime)) throw new Error(`Process ${this.id} adoption identity is invalid`)
|
|
206
|
+
const actualStartTime = processStartTime(pid)
|
|
207
|
+
|
|
208
|
+
if (actualStartTime !== expectedStartTime) throw new Error(`Process ${this.id} adoption rejected: PID ${pid} start time does not match the committed identity`)
|
|
209
|
+
if (this.child) throw new Error(`Process ${this.id} adoption rejected while the process is already supervised`)
|
|
210
|
+
|
|
211
|
+
this.pid = pid
|
|
212
|
+
this.processStartTime = actualStartTime
|
|
213
|
+
this.state = "running"
|
|
214
|
+
this.intentionalStop = false
|
|
215
|
+
this.intentionalStopSignal = undefined
|
|
216
|
+
this.exitCode = undefined
|
|
217
|
+
this.exitSignal = undefined
|
|
218
|
+
this.lastStartReason = "deploy"
|
|
219
|
+
}
|
|
220
|
+
|
|
195
221
|
/**
|
|
196
222
|
* Updates the command template used for future restarts without touching the currently running child.
|
|
197
223
|
* @param {ManagedProcessDefinition} definition - Replacement process definition.
|
|
@@ -264,6 +290,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
264
290
|
this.exitSignal = signal ?? this.intentionalStopSignal
|
|
265
291
|
this.child = undefined
|
|
266
292
|
this.pid = undefined
|
|
293
|
+
this.processStartTime = undefined
|
|
267
294
|
this.exitPromise = undefined
|
|
268
295
|
this.rssBytes = undefined
|
|
269
296
|
this.children = []
|
|
@@ -632,7 +659,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
632
659
|
this.restartTimer = undefined
|
|
633
660
|
}
|
|
634
661
|
if (this.lifecycleRestoreBarrier) await this.lifecycleRestoreBarrier
|
|
635
|
-
if (!this.child?.pid) {
|
|
662
|
+
if (!this.child?.pid && !this.pid) {
|
|
636
663
|
this.state = "stopped"
|
|
637
664
|
if (this.hasRestorableLifecycle()) this.lifecycleRole = "retired"
|
|
638
665
|
return
|
|
@@ -986,6 +1013,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
986
1013
|
logs: this.logs.slice(-this.outputLines),
|
|
987
1014
|
memoryRestarts: this.memoryRestarts,
|
|
988
1015
|
pid: this.pid,
|
|
1016
|
+
processStartTime: this.processStartTime,
|
|
989
1017
|
restarts: this.restarts,
|
|
990
1018
|
retirementTimedOut: this.retirementTimedOut,
|
|
991
1019
|
rssBytes: this.rssBytes,
|
package/src/process-guardian.js
CHANGED
|
@@ -43,6 +43,9 @@ const OWNER_MUTATION_LEASE_MS = 900000
|
|
|
43
43
|
* @property {string} [operation] - Owner mutation diagnostic name.
|
|
44
44
|
* @property {string} [waitingFor] - What the held owner mutation is blocked on.
|
|
45
45
|
* @property {number} [ownerPid] - Exact claimant process PID.
|
|
46
|
+
* @property {string} [configDigest] - Exact committed owner configuration digest.
|
|
47
|
+
* @property {number} [pid] - Exact live process PID to re-adopt.
|
|
48
|
+
* @property {string} [processStartTime] - Exact `/proc` process start-time counter.
|
|
46
49
|
* @property {import("./json.js").JsonValue} [recoverySnapshot] - Public bridge recovery snapshot.
|
|
47
50
|
* @property {string} [statePath] - Exact public recovery state path.
|
|
48
51
|
* @property {import("./json.js").JsonValue} [authority] - Expected current authority.
|
|
@@ -710,6 +713,18 @@ async function execute(request, socket) {
|
|
|
710
713
|
if (request.command === "start") {
|
|
711
714
|
record.desired = true
|
|
712
715
|
await record.process.start(request.reason, request.lifecycleRole)
|
|
716
|
+
} else if (request.command === "adopt") {
|
|
717
|
+
requireOwner(socket, request.command)
|
|
718
|
+
const requestedPid = request.pid
|
|
719
|
+
const requestedStartTime = request.processStartTime
|
|
720
|
+
const committedConfigDigest = ownerState && typeof ownerState === "object" && !Array.isArray(ownerState)
|
|
721
|
+
? /** @type {{authority?: {configDigest?: string}}} */ (ownerState).authority?.configDigest
|
|
722
|
+
: undefined
|
|
723
|
+
if (!request.configDigest || !committedConfigDigest || !isDeepStrictEqual(request.configDigest, committedConfigDigest)) throw new Error("Guardian process adoption authority does not match the committed owner")
|
|
724
|
+
if (!Number.isInteger(requestedPid) || !requestedStartTime) throw new Error("Guardian process adoption requires an exact PID and process start time")
|
|
725
|
+
if (typeof record.process.adopt !== "function") throw new Error("Guardian process adoption is not supported by this process implementation")
|
|
726
|
+
record.desired = true
|
|
727
|
+
await record.process.adopt({pid: /** @type {number} */ (requestedPid), processStartTime: requestedStartTime}, request.configDigest)
|
|
713
728
|
} else if (request.command === "activate") {
|
|
714
729
|
await record.process.activateStrict()
|
|
715
730
|
} else if (request.command === "reactivate" || request.command === "reactivate-with-command") {
|
package/src/process-memory.js
CHANGED
|
@@ -76,6 +76,31 @@ export function processGroupHasLiveMembers(pgid, procPath = "/proc") {
|
|
|
76
76
|
return false
|
|
77
77
|
}
|
|
78
78
|
|
|
79
|
+
/**
|
|
80
|
+
* Returns Linux's monotonic process start-time counter for a PID.
|
|
81
|
+
* @param {number} pid - Process id.
|
|
82
|
+
* @param {string} [procPath] - Procfs root (overridable for deterministic tests).
|
|
83
|
+
* @returns {string | undefined} Kernel start-time ticks, or undefined when unavailable.
|
|
84
|
+
*/
|
|
85
|
+
export function processStartTime(pid, procPath = "/proc") {
|
|
86
|
+
let stat
|
|
87
|
+
|
|
88
|
+
try {
|
|
89
|
+
stat = fs.readFileSync(`${procPath}/${pid}/stat`, "utf8")
|
|
90
|
+
} catch {
|
|
91
|
+
return undefined
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
const commEnd = stat.lastIndexOf(")")
|
|
95
|
+
|
|
96
|
+
if (commEnd < 0) return undefined
|
|
97
|
+
|
|
98
|
+
const fields = stat.slice(commEnd + 2).split(" ")
|
|
99
|
+
const startTime = fields[19]
|
|
100
|
+
|
|
101
|
+
return /^\d+$/.test(startTime || "") ? startTime : undefined
|
|
102
|
+
}
|
|
103
|
+
|
|
79
104
|
/**
|
|
80
105
|
* Measures the total resident memory (RSS) of a managed process group.
|
|
81
106
|
* @param {number} pgid - Process-group id (the detached spawn's pid).
|
package/src/release-group.js
CHANGED
|
@@ -114,6 +114,7 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
114
114
|
this.transferredConnections = /** @type {ReleaseConnections} */ ({http: 0, websocket: 0})
|
|
115
115
|
this.ownerHandoffPaused = false
|
|
116
116
|
this.processes = /** @type {Map<string, ManagedProcess>} */ (new Map())
|
|
117
|
+
this.singletonProcesses = /** @type {Map<string, ManagedProcess>} */ (new Map())
|
|
117
118
|
this.handoffServiceIds = /** @type {Set<string>} */ (new Set())
|
|
118
119
|
this.nonBlockingDrainIds = /** @type {Set<string>} */ (new Set())
|
|
119
120
|
this.ports = /** @type {Record<string, number>} */ ({})
|
|
@@ -186,9 +187,9 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
186
187
|
/**
|
|
187
188
|
* Reconstructs this release around processes still owned by the durable guardian.
|
|
188
189
|
* @param {ReleaseStatus} snapshot - Persisted release snapshot.
|
|
189
|
-
* @param {{synchronizeLifecycleRole?: boolean}} [options] - Guardian role synchronization options.
|
|
190
|
+
* @param {{adoptCommittedGeneration?: boolean, configDigest?: string, synchronizeLifecycleRole?: boolean}} [options] - Guardian role synchronization options.
|
|
190
191
|
*/
|
|
191
|
-
async restore(snapshot, {synchronizeLifecycleRole = true} = {}) {
|
|
192
|
+
async restore(snapshot, {adoptCommittedGeneration = false, configDigest, synchronizeLifecycleRole = true} = {}) {
|
|
192
193
|
if (!snapshot || snapshot.releaseId !== this.releaseId || snapshot.releasePath !== this.releasePath || snapshot.revision !== this.revision) {
|
|
193
194
|
throw new Error(`Persisted release identity mismatch for ${this.releaseId}`)
|
|
194
195
|
}
|
|
@@ -249,7 +250,12 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
249
250
|
this.processes.set(processStatus.id, processInstance)
|
|
250
251
|
if (processConfig.policy === "service" && processConfig.deployStrategy === "handoff") this.handoffServiceIds.add(processStatus.id)
|
|
251
252
|
if (processConfig.nonBlockingDrain) this.nonBlockingDrainIds.add(processStatus.id)
|
|
252
|
-
if ("recover" in processInstance && typeof processInstance.recover === "function")
|
|
253
|
+
if ("recover" in processInstance && typeof processInstance.recover === "function") {
|
|
254
|
+
const adopt = adoptCommittedGeneration && processStatus.state === "stopped" && processStatus.pid && processStatus.processStartTime && configDigest
|
|
255
|
+
? {pid: processStatus.pid, processStartTime: processStatus.processStartTime, configDigest}
|
|
256
|
+
: undefined
|
|
257
|
+
await processInstance.recover(adopt ? {adopt} : undefined)
|
|
258
|
+
}
|
|
253
259
|
}
|
|
254
260
|
if (synchronizeLifecycleRole) await this.synchronizeLifecycleRoles()
|
|
255
261
|
}
|
|
@@ -673,10 +679,16 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
673
679
|
buildProcess(processConfig, options = {}) {
|
|
674
680
|
const definition = this.processDefinition(processConfig, options)
|
|
675
681
|
const instanceId = options.instanceId ?? processConfig.id
|
|
676
|
-
|
|
677
|
-
return this.processFactory
|
|
682
|
+
const process = this.processFactory
|
|
678
683
|
? this.processFactory(options.guardianKey || `release:${this.releaseId}:${instanceId}`, definition)
|
|
679
684
|
: new ManagedProcess(definition)
|
|
685
|
+
|
|
686
|
+
// A singleton is tracked by the daemon rather than in this generation's process map, but its
|
|
687
|
+
// definition is still the one this generation recorded for it. Keeping the instance here is what
|
|
688
|
+
// lets a later retirement move that definition together with the generation's config.
|
|
689
|
+
if (processConfig.policy === "singleton") this.singletonProcesses.set(instanceId, process)
|
|
690
|
+
|
|
691
|
+
return process
|
|
680
692
|
}
|
|
681
693
|
|
|
682
694
|
/**
|
|
@@ -690,7 +702,14 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
690
702
|
this.nonBlockingDrainIds.clear()
|
|
691
703
|
|
|
692
704
|
for (const processConfig of config.processes) {
|
|
693
|
-
const instances = this.getProcesses(processConfig.id)
|
|
705
|
+
const instances = [...this.getProcesses(processConfig.id)]
|
|
706
|
+
|
|
707
|
+
if (processConfig.policy === "singleton") {
|
|
708
|
+
for (const [instanceId, process] of this.singletonProcesses) {
|
|
709
|
+
if (instances.some((instance) => instance.id === instanceId)) continue
|
|
710
|
+
instances.push({id: instanceId, process})
|
|
711
|
+
}
|
|
712
|
+
}
|
|
694
713
|
|
|
695
714
|
for (let index = 0; index < instances.length; index += 1) {
|
|
696
715
|
const instance = instances[index]
|
|
@@ -964,18 +983,19 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
964
983
|
this.drainStartedAt = new Date().toISOString()
|
|
965
984
|
}
|
|
966
985
|
|
|
967
|
-
//
|
|
968
|
-
//
|
|
969
|
-
//
|
|
970
|
-
//
|
|
971
|
-
|
|
972
|
-
|
|
986
|
+
// The retirement policy — which processes are quiesced as one generation, and the drain
|
|
987
|
+
// budget — comes from the caller's config, and the live definitions are refreshed with it.
|
|
988
|
+
// The refreshed config has to become this generation's stored config: the guardian records the
|
|
989
|
+
// refreshed definition's provenance for every process it updates here, and the durable owner
|
|
990
|
+
// state publishes `release.config` as the definition recovery rebuilds each live process from.
|
|
991
|
+
// Keeping the pre-refresh config desynchronises the two, and the next daemon recovery then
|
|
992
|
+
// re-registers a still-live process with the stale provenance and aborts (STOE-2040).
|
|
973
993
|
this.refreshProcessDefinitions(config)
|
|
974
|
-
this.config = generationConfig
|
|
975
994
|
const generationIds = new Set([...this.handoffServiceIds, ...this.nonBlockingDrainIds])
|
|
976
995
|
const results = await Promise.allSettled([...this.processes.entries()]
|
|
977
996
|
.filter(([id]) => generationIds.has(id))
|
|
978
997
|
.map(([, processInstance]) => retry ? processInstance.requiesceStrict() : processInstance.quiesceStrict()))
|
|
998
|
+
|
|
979
999
|
const errors = results.filter((result) => result.status === "rejected").map((result) => result.reason)
|
|
980
1000
|
|
|
981
1001
|
if (errors.length > 0) {
|
|
@@ -101,6 +101,28 @@ test("a known command missing a required field returns a field error", async ()
|
|
|
101
101
|
expect(response.status).toBe("error")
|
|
102
102
|
expect(response.error).toBe("releasePath is required")
|
|
103
103
|
})
|
|
104
|
+
|
|
105
|
+
test("a control connection closed during owner handoff rejects the command", async () => {
|
|
106
|
+
const handoffSocketPath = path.join(root, "handoff.sock")
|
|
107
|
+
const handoffServer = net.createServer((socket) => {
|
|
108
|
+
socket.on("data", () => socket.destroy())
|
|
109
|
+
})
|
|
110
|
+
|
|
111
|
+
await new Promise((resolve, reject) => {
|
|
112
|
+
handoffServer.once("error", reject)
|
|
113
|
+
handoffServer.listen(handoffSocketPath, () => resolve(undefined))
|
|
114
|
+
})
|
|
115
|
+
|
|
116
|
+
try {
|
|
117
|
+
await expect(sendControlCommand({command: {command: "deploy"}, path: handoffSocketPath, timeoutMs: 100})).rejects.toThrow(
|
|
118
|
+
`Rollbridge control connection ${handoffSocketPath} closed before its response`
|
|
119
|
+
)
|
|
120
|
+
} finally {
|
|
121
|
+
await new Promise((resolve) => handoffServer.close(resolve))
|
|
122
|
+
await fs.rm(handoffSocketPath, {force: true})
|
|
123
|
+
}
|
|
124
|
+
})
|
|
125
|
+
|
|
104
126
|
test("status can omit process logs without changing the default status payload", async () => {
|
|
105
127
|
if (!daemon) throw new Error("Missing required fixture: daemon")
|
|
106
128
|
await daemon.deploy({releaseId: "v1", releasePath: root, revision: "v1"})
|
|
@@ -1641,6 +1641,36 @@ test("first upgrade migrates a real pre-split guardian without replacing its own
|
|
|
1641
1641
|
}
|
|
1642
1642
|
})
|
|
1643
1643
|
|
|
1644
|
+
test("a current client re-adopts a stopped record from a legacy guardian", async () => {
|
|
1645
|
+
const fixture = await createLegacyGuardian()
|
|
1646
|
+
const processDefinition = definition("legacy-restore")
|
|
1647
|
+
const legacyProcess = fixture.client.process("release:v1:legacy-restore", processDefinition)
|
|
1648
|
+
let replacement
|
|
1649
|
+
|
|
1650
|
+
try {
|
|
1651
|
+
await legacyProcess.start()
|
|
1652
|
+
const pid = legacyProcess.status().pid
|
|
1653
|
+
|
|
1654
|
+
if (!pid) throw new Error("Missing required legacy process pid")
|
|
1655
|
+
await legacyProcess.stop()
|
|
1656
|
+
await waitForProcessExit(pid)
|
|
1657
|
+
|
|
1658
|
+
replacement = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.client.token})
|
|
1659
|
+
await replacement.connect()
|
|
1660
|
+
const restored = replacement.process("release:v1:legacy-restore", {...processDefinition, env: {RESTORED_BY_CURRENT_CLIENT: "1"}})
|
|
1661
|
+
|
|
1662
|
+
await restored.recover()
|
|
1663
|
+
await restored.start()
|
|
1664
|
+
expect(restored.status().state).toBe("running")
|
|
1665
|
+
} finally {
|
|
1666
|
+
await replacement?.shutdown().catch(() => {})
|
|
1667
|
+
replacement?.disconnect()
|
|
1668
|
+
fixture.client.disconnect()
|
|
1669
|
+
if (fixture.child.exitCode === null && fixture.child.signalCode === null) fixture.child.kill("SIGKILL")
|
|
1670
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
1671
|
+
}
|
|
1672
|
+
})
|
|
1673
|
+
|
|
1644
1674
|
test("split guardian rejects an owner-state update when its nested legacy definition update fails", async () => {
|
|
1645
1675
|
const fixture = await createLegacyGuardian()
|
|
1646
1676
|
const processDefinition = definition("legacy-worker")
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
3
|
import fs from "node:fs"
|
|
4
|
+
import {spawn} from "node:child_process"
|
|
4
5
|
import os from "node:os"
|
|
5
6
|
import path from "node:path"
|
|
6
7
|
import {describe, expect, test} from "@velocious/testing"
|
|
7
8
|
import {fileURLToPath} from "node:url"
|
|
8
9
|
import ManagedProcess, {childRecordIsGone} from "../src/managed-process.js"
|
|
10
|
+
import {processStartTime} from "../src/process-memory.js"
|
|
9
11
|
|
|
10
12
|
describe("managed-process", () => {
|
|
11
13
|
|
|
@@ -111,6 +113,27 @@ test("reports zeroed restart and uptime fields before the process starts", () =>
|
|
|
111
113
|
expect(status.state).toBe("stopped")
|
|
112
114
|
})
|
|
113
115
|
|
|
116
|
+
test("re-adopts a live process whose supervisor status was stopped without spawning a duplicate", async () => {
|
|
117
|
+
const live = spawn(process.execPath, ["-e", "setTimeout(() => {}, 30000)"], {detached: true, stdio: "ignore"})
|
|
118
|
+
await new Promise((resolve, reject) => {
|
|
119
|
+
live.once("error", reject)
|
|
120
|
+
live.once("spawn", resolve)
|
|
121
|
+
})
|
|
122
|
+
if (!live.pid) throw new Error("Missing live process PID")
|
|
123
|
+
const startTime = processStartTime(live.pid)
|
|
124
|
+
if (!startTime) throw new Error("Missing live process start time")
|
|
125
|
+
const managed = buildProcess(10)
|
|
126
|
+
|
|
127
|
+
try {
|
|
128
|
+
expect(managed.status().state).toBe("stopped")
|
|
129
|
+
await managed.adopt({pid: live.pid, processStartTime: startTime}, "committed-config-digest")
|
|
130
|
+
|
|
131
|
+
expect(managed.status()).toMatchObject({pid: live.pid, processStartTime: startTime, state: "running"})
|
|
132
|
+
} finally {
|
|
133
|
+
await managed.stop()
|
|
134
|
+
}
|
|
135
|
+
})
|
|
136
|
+
|
|
114
137
|
test("counts automatic restarts and reports startedAt and uptime while running", async () => {
|
|
115
138
|
const managed = new ManagedProcess({
|
|
116
139
|
command: `${JSON.stringify(process.execPath)} ${JSON.stringify(crasherPath)}`,
|
|
@@ -99,6 +99,30 @@ test("external owner retirement releases guardian authority without losing its g
|
|
|
99
99
|
}
|
|
100
100
|
})
|
|
101
101
|
|
|
102
|
+
test("supervisor shutdown preserves the active generation for guardian owner recovery", async () => {
|
|
103
|
+
const fixture = await createFixture()
|
|
104
|
+
const releasePath = await prepareRelease(fixture.root, "v1")
|
|
105
|
+
const owner = spawnDaemon(fixture.configPath)
|
|
106
|
+
let webPid
|
|
107
|
+
|
|
108
|
+
try {
|
|
109
|
+
await waitForLog(owner, "control socket listening")
|
|
110
|
+
await sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath, revision: "v1"}, path: fixture.socketPath})
|
|
111
|
+
webPid = releaseProcessPid(/** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath})), "v1", "web")
|
|
112
|
+
if (webPid === undefined) throw new Error("The active release did not report a web PID")
|
|
113
|
+
if (owner.pid === undefined) throw new Error("The daemon child did not report a PID")
|
|
114
|
+
|
|
115
|
+
owner.kill("SIGTERM")
|
|
116
|
+
await waitForProcessExit(owner.pid, 5000)
|
|
117
|
+
expect(isProcessRunning(webPid)).toBe(true)
|
|
118
|
+
} finally {
|
|
119
|
+
await sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath}).catch(() => undefined)
|
|
120
|
+
await killChild(owner)
|
|
121
|
+
await stopFixtureGuardian(fixture.statePath)
|
|
122
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
123
|
+
}
|
|
124
|
+
})
|
|
125
|
+
|
|
102
126
|
test("exact bootstrap restores the committed generation after external owner retirement", async () => {
|
|
103
127
|
const fixture = await createFixture()
|
|
104
128
|
const config = normalizeConfig(fixture.config, fixture.configPath)
|
|
@@ -720,7 +744,10 @@ test("guardian recovery becomes ready before replaying a gated generation hook",
|
|
|
720
744
|
await fs.writeFile(retirementGatePath, "release retirement\n")
|
|
721
745
|
await waitForProcessExit(recoveredDaemonPid, 5000)
|
|
722
746
|
await waitForProcessExit(guardianPid, 5000)
|
|
723
|
-
|
|
747
|
+
// The old generation is quiesced only after traffic switched to the candidate, so its quiet
|
|
748
|
+
// hook runs first; the candidate's own quiet hook can only run at shutdown. This is the order
|
|
749
|
+
// the same file asserts at lines 177 and 313.
|
|
750
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2", "retire:v1", "retire:v2"])
|
|
724
751
|
} finally {
|
|
725
752
|
await fs.writeFile(retirementGatePath, "release retirement\n").catch(() => undefined)
|
|
726
753
|
await sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath}).catch(() => undefined)
|
|
@@ -2171,7 +2171,7 @@ async function prepareCandidatePackage(destination, options = {}) {
|
|
|
2171
2171
|
if (dropCommitResponse) {
|
|
2172
2172
|
const clientPath = path.join(destination, "src", "control-client.js")
|
|
2173
2173
|
const source = await fs.readFile(clientPath, "utf8")
|
|
2174
|
-
const marker = "export async function sendControlCommand({command, path}) {\n"
|
|
2174
|
+
const marker = "export async function sendControlCommand({command, path, timeoutMs = DEFAULT_CONTROL_COMMAND_TIMEOUT_MS}) {\n"
|
|
2175
2175
|
const injected = `${marker} if (command.command === "commit-owner-replacement") {\n return await new Promise((resolve, reject) => {\n const socket = net.createConnection(path)\n socket.once("error", reject)\n socket.once("data", () => { socket.destroy(); reject(new Error("injected lost commit response")) })\n socket.once("connect", () => socket.write(\`${"${JSON.stringify(command)}"}\\n\`))\n })\n }\n\n`
|
|
2176
2176
|
const sessionMarker = " this.socket.write(`${JSON.stringify(command)}\\n`)\n return await response\n"
|
|
2177
2177
|
const sessionInjection = " this.socket.write(`${JSON.stringify(command)}\\n`)\n const result = await response\n if (command.command === \"commit-owner-replacement\") throw new Error(\"injected lost commit response\")\n return result\n"
|
|
@@ -556,4 +556,43 @@ test("the retirement transition brackets its own wait and reports a residue it c
|
|
|
556
556
|
fs.rmSync(procPath, {force: true, recursive: true})
|
|
557
557
|
}
|
|
558
558
|
})
|
|
559
|
+
|
|
560
|
+
test("retirement stores the refreshed definition identity it applied to the live generation", async () => {
|
|
561
|
+
const config = normalizeConfig({
|
|
562
|
+
application: "demo",
|
|
563
|
+
control: {path: "/tmp/rollbridge-release-group.sock"},
|
|
564
|
+
processes: [
|
|
565
|
+
{command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}},
|
|
566
|
+
{command: "run jobs", deployStrategy: "handoff", id: "jobs", policy: "service", port: {from: 0, to: 1}}
|
|
567
|
+
],
|
|
568
|
+
proxy: {host: "127.0.0.1", port: 0}
|
|
569
|
+
})
|
|
570
|
+
const release = new ReleaseGroup({config, logger: () => {}, procPath: "/proc", releaseId: "v1", releasePath: "/tmp/rel", revision: "v1", shouldStart: () => true})
|
|
571
|
+
const generationConfig = release.config
|
|
572
|
+
/** @type {Array<import("../src/managed-process.js").ManagedProcessDefinition>} */
|
|
573
|
+
const refreshed = []
|
|
574
|
+
|
|
575
|
+
// eslint-disable-next-line local/no-unknown-jsdoc-type
|
|
576
|
+
release.processes.set("jobs", /** @type {import("../src/managed-process.js").default} */ (/** @type {unknown} */ ({
|
|
577
|
+
id: "jobs:v1",
|
|
578
|
+
quiesceStrict: async () => {},
|
|
579
|
+
status: () => ({pid: 4242, state: "running"}),
|
|
580
|
+
updateDefinition: (/** @type {import("../src/managed-process.js").ManagedProcessDefinition} */ definition) => { refreshed.push(definition) }
|
|
581
|
+
})))
|
|
582
|
+
|
|
583
|
+
const successor = {...release.config, processes: [release.config.processes[0], {...release.config.processes[1], env: {RELEASE_CONFIG_AUTHORITY: "v2"}}]}
|
|
584
|
+
|
|
585
|
+
await release.beginRetirement(successor)
|
|
586
|
+
|
|
587
|
+
// The live process is quiesced with the successor's definition, so the guardian records the
|
|
588
|
+
// successor's provenance for it. The generation's stored config must move with that definition:
|
|
589
|
+
// the durable owner state publishes it as the config recovery rebuilds each live process from, and
|
|
590
|
+
// a retained pre-refresh config makes the next recovery re-register the live process with the
|
|
591
|
+
// stale provenance and abort (STOE-2040).
|
|
592
|
+
expect(refreshed.map((definition) => definition.env?.RELEASE_CONFIG_AUTHORITY)).toEqual(["v2"])
|
|
593
|
+
expect(release.handoffServiceIds.has("jobs")).toBe(true)
|
|
594
|
+
expect(release.config).toBe(successor)
|
|
595
|
+
expect(release.config).not.toBe(generationConfig)
|
|
596
|
+
expect(release.state).toBe("draining")
|
|
597
|
+
})
|
|
559
598
|
})
|