rollbridge 0.1.74 → 0.1.75

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,4 @@
1
+ ### Fixed
2
+
3
+ - Make a control command fail when its owner connection closes before a response,
4
+ and bound commands that never receive a response.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "rollbridge",
3
- "version": "0.1.74",
3
+ "version": "0.1.75",
4
4
  "description": "Zero-downtime process supervisor and local traffic switcher for deploy-managed apps.",
5
5
  "keywords": [
6
6
  "deploy",
package/src/cli.js CHANGED
@@ -82,7 +82,7 @@ export async function runCli(argv) {
82
82
  const shutdown = async () => {
83
83
  shutdownRequested = true
84
84
  await startupPromise.catch(() => undefined)
85
- await daemon.shutdown()
85
+ await daemon.shutdown({preserveActiveRelease: Boolean(config.ownerRecovery)})
86
86
  process.exit(0)
87
87
  }
88
88
 
@@ -2,6 +2,8 @@
2
2
 
3
3
  import net from "node:net"
4
4
 
5
+ const DEFAULT_CONTROL_COMMAND_TIMEOUT_MS = 30000
6
+
5
7
  /**
6
8
  * @typedef {import("./json.js").JsonValue} JsonValue
7
9
  */
@@ -11,15 +13,40 @@ import net from "node:net"
11
13
  * @param {object} args - Options.
12
14
  * @param {Record<string, JsonValue>} args.command - Command payload.
13
15
  * @param {string} args.path - Control socket path.
16
+ * @param {number} [args.timeoutMs] - Maximum time to wait for a response.
14
17
  * @returns {Promise<Record<string, JsonValue>>} Response payload.
15
18
  */
16
- export async function sendControlCommand({command, path}) {
19
+ export async function sendControlCommand({command, path, timeoutMs = DEFAULT_CONTROL_COMMAND_TIMEOUT_MS}) {
17
20
  return await new Promise((resolve, reject) => {
18
21
  const socket = net.createConnection(path)
19
22
  let buffer = ""
23
+ let settled = false
24
+ const timeout = setTimeout(() => {
25
+ socket.destroy()
26
+ settleReject(new Error(`Rollbridge control command ${String(command.command)} on ${path} timed out after ${timeoutMs}ms`))
27
+ }, timeoutMs)
28
+
29
+ /** @param {Error} error - Command failure. */
30
+ const settleReject = (error) => {
31
+ if (settled) return
32
+ settled = true
33
+ clearTimeout(timeout)
34
+ reject(error)
35
+ }
36
+
37
+ /** @param {Record<string, JsonValue>} response - Command response. */
38
+ const settleResolve = (response) => {
39
+ if (settled) return
40
+ settled = true
41
+ clearTimeout(timeout)
42
+ resolve(response)
43
+ }
20
44
 
21
45
  socket.setEncoding("utf8")
22
- socket.once("error", reject)
46
+ socket.once("error", settleReject)
47
+ socket.once("close", () => {
48
+ if (!settled) settleReject(new Error(`Rollbridge control connection ${path} closed before its response`))
49
+ })
23
50
  socket.on("data", (chunk) => {
24
51
  buffer += chunk
25
52
  const newlineIndex = buffer.indexOf("\n")
@@ -33,16 +60,16 @@ export async function sendControlCommand({command, path}) {
33
60
  response = JSON.parse(line)
34
61
  } catch (error) {
35
62
  socket.destroy()
36
- reject(new Error(`Invalid Rollbridge control response from ${path}`, {cause: error}))
63
+ settleReject(new Error(`Invalid Rollbridge control response from ${path}`, {cause: error}))
37
64
  return
38
65
  }
39
66
 
40
67
  socket.end()
41
68
 
42
69
  if (response.status === "error") {
43
- reject(new Error(String(response.error || "Unknown Rollbridge error")))
70
+ settleReject(new Error(String(response.error || "Unknown Rollbridge error")))
44
71
  } else {
45
- resolve(response)
72
+ settleResolve(response)
46
73
  }
47
74
  })
48
75
  socket.write(`${JSON.stringify(command)}\n`)
package/src/daemon.js CHANGED
@@ -25,7 +25,7 @@ const STATE_PERSIST_INTERVAL_MS = 5000
25
25
  * @typedef {{releaseId?: string, releasePath: string, revision?: string}} DeployArgs
26
26
  * @typedef {{attestation?: string, releaseId: string, releasePath: string, revision: string}} BootstrapIdentity
27
27
  * @typedef {{id: string, process: import("./managed-process.js").ManagedProcessStatus}} ProcessStatus
28
- * @typedef {{disruptive: true, mode: "guardian-version-handoff" | "legacy-first-upgrade", reason: string}} OwnerTransition
28
+ * @typedef {{disruptive: boolean, mode: "guardian-version-handoff" | "legacy-first-upgrade", reason: string}} OwnerTransition
29
29
  * @typedef {"guardian-version-drift" | "legacy-protocol"} GuardianHandoffProtocol
30
30
  * @typedef {"candidate_ready" | "retiring_previous" | "previous_retired" | "activating_candidate" | "restoring_previous" | "retiring_failed_candidate" | "degraded_active" | "committed_pending" | "committed" | "restoring_committed"} GenerationTransitionPhase
31
31
  * @typedef {{activationError?: string, activationLifecycle?: boolean, candidateReleaseId: string, candidateReleasePath: string, candidateRevision: string, compensationError?: string, configDigest: string, degradedIncumbent?: boolean, error?: string, journalRevision?: number, phase: GenerationTransitionPhase, previousReleaseId: string | null, startedAt: string, updatedAt: string}} GenerationTransition
@@ -412,7 +412,11 @@ export default class RollbridgeDaemon {
412
412
  shouldStart: () => !this.stopping
413
413
  })
414
414
 
415
- await release.restore(releaseStatus, {synchronizeLifecycleRole: synchronizeLifecycleRoles})
415
+ await release.restore(releaseStatus, {
416
+ adoptCommittedGeneration: releaseStatus.releaseId === snapshot.activeReleaseId && snapshot.generationTransition?.phase === "committed" && snapshot.ownerTransition?.mode === "guardian-version-handoff",
417
+ configDigest: ownerConfigDigest(config),
418
+ synchronizeLifecycleRole: synchronizeLifecycleRoles
419
+ })
416
420
  this.releases.set(release.releaseId, release)
417
421
  if (release.releaseId === snapshot.activeReleaseId) this.activeRelease = release
418
422
  }
@@ -3051,17 +3055,52 @@ export default class RollbridgeDaemon {
3051
3055
 
3052
3056
  /**
3053
3057
  * Stops proxy, control socket, and child processes.
3054
- * @param {{completionSocket?: net.Socket, waitForControlConnections?: boolean}} [options] - Shutdown connection behavior.
3058
+ * @param {{completionSocket?: net.Socket, preserveActiveRelease?: boolean, waitForControlConnections?: boolean}} [options] - Shutdown connection behavior.
3055
3059
  * @returns {Promise<void>} Resolves when owned resources are stopped (and, by default, control connections close).
3056
3060
  */
3057
- async shutdown({completionSocket, waitForControlConnections = true} = {}) {
3058
- this.assertNoGenerationTransitionRecovery("shut down")
3059
- if (!this.shutdownPromise) this.shutdownPromise = this.performShutdown(completionSocket)
3061
+ async shutdown({completionSocket, preserveActiveRelease = false, waitForControlConnections = true} = {}) {
3062
+ // A supervisor signal during committed_pending is after candidate activation: the candidate
3063
+ // is already serving, so the recovery shutdown may finish the bounded retirement. Earlier
3064
+ // transition phases still refuse shutdown rather than risking the active incumbent.
3065
+ const finishCommittedPendingRetirement = preserveActiveRelease && this.generationTransition?.phase === "committed_pending"
3066
+
3067
+ if (!finishCommittedPendingRetirement) this.assertNoGenerationTransitionRecovery("shut down")
3068
+ if (!this.shutdownPromise) {
3069
+ this.shutdownPromise = finishCommittedPendingRetirement
3070
+ ? this.performShutdown(completionSocket)
3071
+ : preserveActiveRelease && this.guardian
3072
+ ? this.performRecoveryShutdown(completionSocket)
3073
+ : this.performShutdown(completionSocket)
3074
+ }
3060
3075
 
3061
3076
  await this.shutdownPromise
3062
3077
  if (waitForControlConnections && this.controlClosePromise) await this.controlClosePromise
3063
3078
  }
3064
3079
 
3080
+ /**
3081
+ * Relinquishes this daemon during supervisor recovery without stopping guardian-owned
3082
+ * generations. The guardian will start or reconnect the replacement owner, which is the only
3083
+ * point at which the active release may be retired.
3084
+ * @param {net.Socket | undefined} completionSocket - Requesting connection, if any.
3085
+ * @returns {Promise<void>} Resolves after this daemon's listeners are released.
3086
+ */
3087
+ async performRecoveryShutdown(completionSocket) {
3088
+ this.ownerRetired = true
3089
+ this.controlCommandsReady = false
3090
+ this.stopping = true
3091
+ if (this.persistTimer) clearInterval(this.persistTimer)
3092
+ this.persistTimer = undefined
3093
+ if (this.pendingWrite) await this.pendingWrite
3094
+ this.persistenceEnabled = false
3095
+ this.stateCleanupEnabled = false
3096
+ this.controlClosePromise = this.closeServer(this.controlServer)
3097
+ this.proxyClosePromise = this.closeServer(this.proxyServer)
3098
+ for (const socket of this.controlSockets) if (socket !== completionSocket) socket.destroy()
3099
+ await this.removeControlSocket()
3100
+ await Promise.all([this.controlClosePromise, this.proxyClosePromise])
3101
+ this.guardian?.disconnect()
3102
+ }
3103
+
3065
3104
  /**
3066
3105
  * Relinquishes stable listeners promptly while retaining draining children under
3067
3106
  * this daemon until their normal stop contract completes.
@@ -516,18 +516,56 @@ class GuardianProcess extends ManagedProcess {
516
516
  if (!this.registration) {
517
517
  this.registration = this.client.request({command: "register", definition: this.definition, key: this.key, provenance: this.provenance})
518
518
  .then((status) => { this.cachedStatus = asProcessStatus(status) })
519
+ .catch(async (error) => {
520
+ // Guardians predating stale-record re-registration reject a restore before
521
+ // the current guardian can apply its stopped/failed-record fence. Reconcile
522
+ // only an exact, conclusively dead record through the authenticated remove
523
+ // command; live or ambiguous records remain fail-closed on provenance.
524
+ if (!(error instanceof Error) || !error.message.startsWith(`Guardian provenance mismatch for ${this.key}`)) throw error
525
+ const stale = (await this.client.inventory()).find((entry) => entry.key === this.key)
526
+ const staleStatus = stale?.status
527
+ const staleIsGone = staleStatus &&
528
+ (staleStatus.state === "stopped" || staleStatus.state === "failed") &&
529
+ staleStatus.pid == null &&
530
+ (!staleStatus.heldProcessGroup || staleStatus.heldProcessGroup.liveMemberCount === 0)
531
+
532
+ if (!stale || !staleIsGone) throw error
533
+ await this.client.remove(stale.key, stale.provenance)
534
+ const status = await this.client.request({command: "register", definition: this.definition, key: this.key, provenance: this.provenance})
535
+
536
+ this.cachedStatus = asProcessStatus(status)
537
+ })
538
+ }
539
+ try {
540
+ await this.registration
541
+ } catch (error) {
542
+ this.registration = undefined
543
+ throw error
519
544
  }
520
- await this.registration
521
545
  }
522
546
 
523
547
  /** Reconnects to an already registered guardian process without changing its desired state. */
524
- async recover() {
548
+ /** @param {{adopt?: {configDigest: string, pid: number, processStartTime: string}}} [options] - Optional identity-fenced adoption. */
549
+ async recover({adopt} = {}) {
525
550
  if (this.client.reservedProcessKey === this.key) {
526
551
  if (this.client.reservedProcessProvenance !== this.provenance) throw new Error(`Guardian provenance mismatch for reserved process ${this.key}`)
527
552
  this.cachedStatus = asProcessStatus(await this.client.request({command: "status", key: this.key}))
553
+ if (adopt) await this.adopt(adopt, adopt.configDigest)
528
554
  return
529
555
  }
530
556
  await this.ensureRegistered()
557
+ if (adopt) await this.adopt(adopt, adopt.configDigest)
558
+ }
559
+
560
+ /** Re-adopts a live committed process after a guardian/version handoff. */
561
+ /**
562
+ * @param {{pid: number, processStartTime: string}} identity - Exact process identity.
563
+ * @param {string} [configDigest] - Committed owner config digest.
564
+ */
565
+ async adopt(identity, configDigest) {
566
+ if (!configDigest) throw new Error("Guardian process adoption requires the committed owner config digest")
567
+ await this.ensureRegistered()
568
+ this.cachedStatus = asProcessStatus(await this.client.request({command: "adopt", key: this.key, pid: identity.pid, processStartTime: identity.processStartTime, configDigest}))
531
569
  }
532
570
 
533
571
  /** Attaches a reconstructed process whose incumbent-owned registration was reserved through commit. */
@@ -2,7 +2,7 @@
2
2
 
3
3
  import {EventEmitter} from "node:events"
4
4
  import {spawn} from "node:child_process"
5
- import {processGroupHasLiveMembers, processGroupMembers, sampleProcessGroup} from "./process-memory.js"
5
+ import {processGroupHasLiveMembers, processGroupMembers, processStartTime, sampleProcessGroup} from "./process-memory.js"
6
6
 
7
7
  const DEFAULT_ACTIVATION_HOOK_TIMEOUT_MS = 30000
8
8
  const MAX_BUFFERED_OUTPUT_CHARACTERS = 64 * 1024
@@ -24,7 +24,7 @@ const ESCALATION_KILL_CONFIRM_MS = 5000
24
24
  * @typedef {import("./config.js").StopTimeoutMs} StopTimeoutMs
25
25
  * @typedef {{command: string, cwd: string | undefined, env: Record<string, string | undefined>, lifecycle: import("./config.js").LifecycleConfig, logger: (message: string, data?: Record<string, import("./json.js").JsonValue>) => void, memory: import("./config.js").MemoryConfig | undefined, outputLines: number, restart: import("./config.js").RestartConfig, restartDelayMs: number, shouldRestart: () => boolean, shouldStop?: () => boolean, stopSignal: string, stopTimeoutMs: StopTimeoutMs}} ManagedProcessDefinition
26
26
  * @typedef {{liveMemberCount: number | undefined, pgid: number}} HeldProcessGroup
27
- * @typedef {{children: import("./process-memory.js").ProcessGroupMember[], command: string, cwd: string | undefined, exitCode: number | null | undefined, exitSignal: ProcessExitSignal | undefined, heldProcessGroup: HeldProcessGroup | undefined, id: string, lastMemoryRestartAt: string | undefined, lastStartReason: ManagedProcessStartReason | undefined, lifecycleRole?: LifecycleRole, logs: ManagedProcessLog[], memoryRestarts: number, pid: number | undefined, restarts: number, retirementTimedOut: boolean, rssBytes: number | undefined, startedAt: string | undefined, state: ManagedProcessState, uptimeMs: number | undefined}} ManagedProcessStatus
27
+ * @typedef {{children: import("./process-memory.js").ProcessGroupMember[], command: string, cwd: string | undefined, exitCode: number | null | undefined, exitSignal: ProcessExitSignal | undefined, heldProcessGroup: HeldProcessGroup | undefined, id: string, lastMemoryRestartAt: string | undefined, lastStartReason: ManagedProcessStartReason | undefined, lifecycleRole?: LifecycleRole, logs: ManagedProcessLog[], memoryRestarts: number, pid: number | undefined, processStartTime?: string, restarts: number, retirementTimedOut: boolean, rssBytes: number | undefined, startedAt: string | undefined, state: ManagedProcessState, uptimeMs: number | undefined}} ManagedProcessStatus
28
28
  */
29
29
 
30
30
  export default class ManagedProcess extends EventEmitter {
@@ -92,6 +92,7 @@ export default class ManagedProcess extends EventEmitter {
92
92
  this.child = undefined
93
93
  this.exitPromise = undefined
94
94
  this.pid = undefined
95
+ this.processStartTime = undefined
95
96
  this.exitCode = undefined
96
97
  this.exitSignal = undefined
97
98
  }
@@ -144,6 +145,7 @@ export default class ManagedProcess extends EventEmitter {
144
145
  child.once("spawn", () => {
145
146
  void (async () => {
146
147
  this.startedAtMs = Date.now()
148
+ this.processStartTime = child.pid ? processStartTime(child.pid) : undefined
147
149
  this.lastStartReason = reason
148
150
  const lifecycleRestore = this.restoreLifecycleRole()
149
151
  const lifecycleRestoreBarrier = lifecycleRestore.then(() => undefined, () => undefined)
@@ -179,6 +181,7 @@ export default class ManagedProcess extends EventEmitter {
179
181
  if (this.child === child) {
180
182
  this.child = undefined
181
183
  this.pid = undefined
184
+ this.processStartTime = undefined
182
185
  this.exitPromise = undefined
183
186
  }
184
187
  reject(error)
@@ -192,6 +195,29 @@ export default class ManagedProcess extends EventEmitter {
192
195
  })
193
196
  }
194
197
 
198
+ /**
199
+ * Re-adopts an already-live process after a guardian/version handoff.
200
+ * @param {{pid: number, processStartTime: string}} identity - Exact process identity.
201
+ * @param {string} [_configDigest] - Owner digest accepted by guardian-backed implementations.
202
+ * @returns {Promise<void>}
203
+ */
204
+ async adopt({pid, processStartTime: expectedStartTime}, _configDigest) {
205
+ if (!Number.isInteger(pid) || pid <= 0 || !/^\d+$/.test(expectedStartTime)) throw new Error(`Process ${this.id} adoption identity is invalid`)
206
+ const actualStartTime = processStartTime(pid)
207
+
208
+ if (actualStartTime !== expectedStartTime) throw new Error(`Process ${this.id} adoption rejected: PID ${pid} start time does not match the committed identity`)
209
+ if (this.child) throw new Error(`Process ${this.id} adoption rejected while the process is already supervised`)
210
+
211
+ this.pid = pid
212
+ this.processStartTime = actualStartTime
213
+ this.state = "running"
214
+ this.intentionalStop = false
215
+ this.intentionalStopSignal = undefined
216
+ this.exitCode = undefined
217
+ this.exitSignal = undefined
218
+ this.lastStartReason = "deploy"
219
+ }
220
+
195
221
  /**
196
222
  * Updates the command template used for future restarts without touching the currently running child.
197
223
  * @param {ManagedProcessDefinition} definition - Replacement process definition.
@@ -264,6 +290,7 @@ export default class ManagedProcess extends EventEmitter {
264
290
  this.exitSignal = signal ?? this.intentionalStopSignal
265
291
  this.child = undefined
266
292
  this.pid = undefined
293
+ this.processStartTime = undefined
267
294
  this.exitPromise = undefined
268
295
  this.rssBytes = undefined
269
296
  this.children = []
@@ -632,7 +659,7 @@ export default class ManagedProcess extends EventEmitter {
632
659
  this.restartTimer = undefined
633
660
  }
634
661
  if (this.lifecycleRestoreBarrier) await this.lifecycleRestoreBarrier
635
- if (!this.child?.pid) {
662
+ if (!this.child?.pid && !this.pid) {
636
663
  this.state = "stopped"
637
664
  if (this.hasRestorableLifecycle()) this.lifecycleRole = "retired"
638
665
  return
@@ -986,6 +1013,7 @@ export default class ManagedProcess extends EventEmitter {
986
1013
  logs: this.logs.slice(-this.outputLines),
987
1014
  memoryRestarts: this.memoryRestarts,
988
1015
  pid: this.pid,
1016
+ processStartTime: this.processStartTime,
989
1017
  restarts: this.restarts,
990
1018
  retirementTimedOut: this.retirementTimedOut,
991
1019
  rssBytes: this.rssBytes,
@@ -43,6 +43,9 @@ const OWNER_MUTATION_LEASE_MS = 900000
43
43
  * @property {string} [operation] - Owner mutation diagnostic name.
44
44
  * @property {string} [waitingFor] - What the held owner mutation is blocked on.
45
45
  * @property {number} [ownerPid] - Exact claimant process PID.
46
+ * @property {string} [configDigest] - Exact committed owner configuration digest.
47
+ * @property {number} [pid] - Exact live process PID to re-adopt.
48
+ * @property {string} [processStartTime] - Exact `/proc` process start-time counter.
46
49
  * @property {import("./json.js").JsonValue} [recoverySnapshot] - Public bridge recovery snapshot.
47
50
  * @property {string} [statePath] - Exact public recovery state path.
48
51
  * @property {import("./json.js").JsonValue} [authority] - Expected current authority.
@@ -710,6 +713,18 @@ async function execute(request, socket) {
710
713
  if (request.command === "start") {
711
714
  record.desired = true
712
715
  await record.process.start(request.reason, request.lifecycleRole)
716
+ } else if (request.command === "adopt") {
717
+ requireOwner(socket, request.command)
718
+ const requestedPid = request.pid
719
+ const requestedStartTime = request.processStartTime
720
+ const committedConfigDigest = ownerState && typeof ownerState === "object" && !Array.isArray(ownerState)
721
+ ? /** @type {{authority?: {configDigest?: string}}} */ (ownerState).authority?.configDigest
722
+ : undefined
723
+ if (!request.configDigest || !committedConfigDigest || !isDeepStrictEqual(request.configDigest, committedConfigDigest)) throw new Error("Guardian process adoption authority does not match the committed owner")
724
+ if (!Number.isInteger(requestedPid) || !requestedStartTime) throw new Error("Guardian process adoption requires an exact PID and process start time")
725
+ if (typeof record.process.adopt !== "function") throw new Error("Guardian process adoption is not supported by this process implementation")
726
+ record.desired = true
727
+ await record.process.adopt({pid: /** @type {number} */ (requestedPid), processStartTime: requestedStartTime}, request.configDigest)
713
728
  } else if (request.command === "activate") {
714
729
  await record.process.activateStrict()
715
730
  } else if (request.command === "reactivate" || request.command === "reactivate-with-command") {
@@ -76,6 +76,31 @@ export function processGroupHasLiveMembers(pgid, procPath = "/proc") {
76
76
  return false
77
77
  }
78
78
 
79
+ /**
80
+ * Returns Linux's monotonic process start-time counter for a PID.
81
+ * @param {number} pid - Process id.
82
+ * @param {string} [procPath] - Procfs root (overridable for deterministic tests).
83
+ * @returns {string | undefined} Kernel start-time ticks, or undefined when unavailable.
84
+ */
85
+ export function processStartTime(pid, procPath = "/proc") {
86
+ let stat
87
+
88
+ try {
89
+ stat = fs.readFileSync(`${procPath}/${pid}/stat`, "utf8")
90
+ } catch {
91
+ return undefined
92
+ }
93
+
94
+ const commEnd = stat.lastIndexOf(")")
95
+
96
+ if (commEnd < 0) return undefined
97
+
98
+ const fields = stat.slice(commEnd + 2).split(" ")
99
+ const startTime = fields[19]
100
+
101
+ return /^\d+$/.test(startTime || "") ? startTime : undefined
102
+ }
103
+
79
104
  /**
80
105
  * Measures the total resident memory (RSS) of a managed process group.
81
106
  * @param {number} pgid - Process-group id (the detached spawn's pid).
@@ -114,6 +114,7 @@ export default class ReleaseGroup extends EventEmitter {
114
114
  this.transferredConnections = /** @type {ReleaseConnections} */ ({http: 0, websocket: 0})
115
115
  this.ownerHandoffPaused = false
116
116
  this.processes = /** @type {Map<string, ManagedProcess>} */ (new Map())
117
+ this.singletonProcesses = /** @type {Map<string, ManagedProcess>} */ (new Map())
117
118
  this.handoffServiceIds = /** @type {Set<string>} */ (new Set())
118
119
  this.nonBlockingDrainIds = /** @type {Set<string>} */ (new Set())
119
120
  this.ports = /** @type {Record<string, number>} */ ({})
@@ -186,9 +187,9 @@ export default class ReleaseGroup extends EventEmitter {
186
187
  /**
187
188
  * Reconstructs this release around processes still owned by the durable guardian.
188
189
  * @param {ReleaseStatus} snapshot - Persisted release snapshot.
189
- * @param {{synchronizeLifecycleRole?: boolean}} [options] - Guardian role synchronization options.
190
+ * @param {{adoptCommittedGeneration?: boolean, configDigest?: string, synchronizeLifecycleRole?: boolean}} [options] - Guardian role synchronization options.
190
191
  */
191
- async restore(snapshot, {synchronizeLifecycleRole = true} = {}) {
192
+ async restore(snapshot, {adoptCommittedGeneration = false, configDigest, synchronizeLifecycleRole = true} = {}) {
192
193
  if (!snapshot || snapshot.releaseId !== this.releaseId || snapshot.releasePath !== this.releasePath || snapshot.revision !== this.revision) {
193
194
  throw new Error(`Persisted release identity mismatch for ${this.releaseId}`)
194
195
  }
@@ -249,7 +250,12 @@ export default class ReleaseGroup extends EventEmitter {
249
250
  this.processes.set(processStatus.id, processInstance)
250
251
  if (processConfig.policy === "service" && processConfig.deployStrategy === "handoff") this.handoffServiceIds.add(processStatus.id)
251
252
  if (processConfig.nonBlockingDrain) this.nonBlockingDrainIds.add(processStatus.id)
252
- if ("recover" in processInstance && typeof processInstance.recover === "function") await processInstance.recover()
253
+ if ("recover" in processInstance && typeof processInstance.recover === "function") {
254
+ const adopt = adoptCommittedGeneration && processStatus.state === "stopped" && processStatus.pid && processStatus.processStartTime && configDigest
255
+ ? {pid: processStatus.pid, processStartTime: processStatus.processStartTime, configDigest}
256
+ : undefined
257
+ await processInstance.recover(adopt ? {adopt} : undefined)
258
+ }
253
259
  }
254
260
  if (synchronizeLifecycleRole) await this.synchronizeLifecycleRoles()
255
261
  }
@@ -673,10 +679,16 @@ export default class ReleaseGroup extends EventEmitter {
673
679
  buildProcess(processConfig, options = {}) {
674
680
  const definition = this.processDefinition(processConfig, options)
675
681
  const instanceId = options.instanceId ?? processConfig.id
676
-
677
- return this.processFactory
682
+ const process = this.processFactory
678
683
  ? this.processFactory(options.guardianKey || `release:${this.releaseId}:${instanceId}`, definition)
679
684
  : new ManagedProcess(definition)
685
+
686
+ // A singleton is tracked by the daemon rather than in this generation's process map, but its
687
+ // definition is still the one this generation recorded for it. Keeping the instance here is what
688
+ // lets a later retirement move that definition together with the generation's config.
689
+ if (processConfig.policy === "singleton") this.singletonProcesses.set(instanceId, process)
690
+
691
+ return process
680
692
  }
681
693
 
682
694
  /**
@@ -690,7 +702,14 @@ export default class ReleaseGroup extends EventEmitter {
690
702
  this.nonBlockingDrainIds.clear()
691
703
 
692
704
  for (const processConfig of config.processes) {
693
- const instances = this.getProcesses(processConfig.id)
705
+ const instances = [...this.getProcesses(processConfig.id)]
706
+
707
+ if (processConfig.policy === "singleton") {
708
+ for (const [instanceId, process] of this.singletonProcesses) {
709
+ if (instances.some((instance) => instance.id === instanceId)) continue
710
+ instances.push({id: instanceId, process})
711
+ }
712
+ }
694
713
 
695
714
  for (let index = 0; index < instances.length; index += 1) {
696
715
  const instance = instances[index]
@@ -964,18 +983,19 @@ export default class ReleaseGroup extends EventEmitter {
964
983
  this.drainStartedAt = new Date().toISOString()
965
984
  }
966
985
 
967
- // Refresh the live process definitions with the current retirement policy,
968
- // but retain this generation's exact config identity for owner recovery.
969
- // Replacing the stored generation config here would make a still-running
970
- // singleton from the previous release fail the guardian provenance fence.
971
- const generationConfig = this.config
972
-
986
+ // The retirement policy — which processes are quiesced as one generation, and the drain
987
+ // budget — comes from the caller's config, and the live definitions are refreshed with it.
988
+ // The refreshed config has to become this generation's stored config: the guardian records the
989
+ // refreshed definition's provenance for every process it updates here, and the durable owner
990
+ // state publishes `release.config` as the definition recovery rebuilds each live process from.
991
+ // Keeping the pre-refresh config desynchronises the two, and the next daemon recovery then
992
+ // re-registers a still-live process with the stale provenance and aborts (STOE-2040).
973
993
  this.refreshProcessDefinitions(config)
974
- this.config = generationConfig
975
994
  const generationIds = new Set([...this.handoffServiceIds, ...this.nonBlockingDrainIds])
976
995
  const results = await Promise.allSettled([...this.processes.entries()]
977
996
  .filter(([id]) => generationIds.has(id))
978
997
  .map(([, processInstance]) => retry ? processInstance.requiesceStrict() : processInstance.quiesceStrict()))
998
+
979
999
  const errors = results.filter((result) => result.status === "rejected").map((result) => result.reason)
980
1000
 
981
1001
  if (errors.length > 0) {
@@ -101,6 +101,28 @@ test("a known command missing a required field returns a field error", async ()
101
101
  expect(response.status).toBe("error")
102
102
  expect(response.error).toBe("releasePath is required")
103
103
  })
104
+
105
+ test("a control connection closed during owner handoff rejects the command", async () => {
106
+ const handoffSocketPath = path.join(root, "handoff.sock")
107
+ const handoffServer = net.createServer((socket) => {
108
+ socket.on("data", () => socket.destroy())
109
+ })
110
+
111
+ await new Promise((resolve, reject) => {
112
+ handoffServer.once("error", reject)
113
+ handoffServer.listen(handoffSocketPath, () => resolve(undefined))
114
+ })
115
+
116
+ try {
117
+ await expect(sendControlCommand({command: {command: "deploy"}, path: handoffSocketPath, timeoutMs: 100})).rejects.toThrow(
118
+ `Rollbridge control connection ${handoffSocketPath} closed before its response`
119
+ )
120
+ } finally {
121
+ await new Promise((resolve) => handoffServer.close(resolve))
122
+ await fs.rm(handoffSocketPath, {force: true})
123
+ }
124
+ })
125
+
104
126
  test("status can omit process logs without changing the default status payload", async () => {
105
127
  if (!daemon) throw new Error("Missing required fixture: daemon")
106
128
  await daemon.deploy({releaseId: "v1", releasePath: root, revision: "v1"})
@@ -1641,6 +1641,36 @@ test("first upgrade migrates a real pre-split guardian without replacing its own
1641
1641
  }
1642
1642
  })
1643
1643
 
1644
+ test("a current client re-adopts a stopped record from a legacy guardian", async () => {
1645
+ const fixture = await createLegacyGuardian()
1646
+ const processDefinition = definition("legacy-restore")
1647
+ const legacyProcess = fixture.client.process("release:v1:legacy-restore", processDefinition)
1648
+ let replacement
1649
+
1650
+ try {
1651
+ await legacyProcess.start()
1652
+ const pid = legacyProcess.status().pid
1653
+
1654
+ if (!pid) throw new Error("Missing required legacy process pid")
1655
+ await legacyProcess.stop()
1656
+ await waitForProcessExit(pid)
1657
+
1658
+ replacement = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.client.token})
1659
+ await replacement.connect()
1660
+ const restored = replacement.process("release:v1:legacy-restore", {...processDefinition, env: {RESTORED_BY_CURRENT_CLIENT: "1"}})
1661
+
1662
+ await restored.recover()
1663
+ await restored.start()
1664
+ expect(restored.status().state).toBe("running")
1665
+ } finally {
1666
+ await replacement?.shutdown().catch(() => {})
1667
+ replacement?.disconnect()
1668
+ fixture.client.disconnect()
1669
+ if (fixture.child.exitCode === null && fixture.child.signalCode === null) fixture.child.kill("SIGKILL")
1670
+ await fs.rm(fixture.root, {force: true, recursive: true})
1671
+ }
1672
+ })
1673
+
1644
1674
  test("split guardian rejects an owner-state update when its nested legacy definition update fails", async () => {
1645
1675
  const fixture = await createLegacyGuardian()
1646
1676
  const processDefinition = definition("legacy-worker")
@@ -1,11 +1,13 @@
1
1
  // @ts-check
2
2
 
3
3
  import fs from "node:fs"
4
+ import {spawn} from "node:child_process"
4
5
  import os from "node:os"
5
6
  import path from "node:path"
6
7
  import {describe, expect, test} from "@velocious/testing"
7
8
  import {fileURLToPath} from "node:url"
8
9
  import ManagedProcess, {childRecordIsGone} from "../src/managed-process.js"
10
+ import {processStartTime} from "../src/process-memory.js"
9
11
 
10
12
  describe("managed-process", () => {
11
13
 
@@ -111,6 +113,27 @@ test("reports zeroed restart and uptime fields before the process starts", () =>
111
113
  expect(status.state).toBe("stopped")
112
114
  })
113
115
 
116
+ test("re-adopts a live process whose supervisor status was stopped without spawning a duplicate", async () => {
117
+ const live = spawn(process.execPath, ["-e", "setTimeout(() => {}, 30000)"], {detached: true, stdio: "ignore"})
118
+ await new Promise((resolve, reject) => {
119
+ live.once("error", reject)
120
+ live.once("spawn", resolve)
121
+ })
122
+ if (!live.pid) throw new Error("Missing live process PID")
123
+ const startTime = processStartTime(live.pid)
124
+ if (!startTime) throw new Error("Missing live process start time")
125
+ const managed = buildProcess(10)
126
+
127
+ try {
128
+ expect(managed.status().state).toBe("stopped")
129
+ await managed.adopt({pid: live.pid, processStartTime: startTime}, "committed-config-digest")
130
+
131
+ expect(managed.status()).toMatchObject({pid: live.pid, processStartTime: startTime, state: "running"})
132
+ } finally {
133
+ await managed.stop()
134
+ }
135
+ })
136
+
114
137
  test("counts automatic restarts and reports startedAt and uptime while running", async () => {
115
138
  const managed = new ManagedProcess({
116
139
  command: `${JSON.stringify(process.execPath)} ${JSON.stringify(crasherPath)}`,
@@ -99,6 +99,30 @@ test("external owner retirement releases guardian authority without losing its g
99
99
  }
100
100
  })
101
101
 
102
+ test("supervisor shutdown preserves the active generation for guardian owner recovery", async () => {
103
+ const fixture = await createFixture()
104
+ const releasePath = await prepareRelease(fixture.root, "v1")
105
+ const owner = spawnDaemon(fixture.configPath)
106
+ let webPid
107
+
108
+ try {
109
+ await waitForLog(owner, "control socket listening")
110
+ await sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath, revision: "v1"}, path: fixture.socketPath})
111
+ webPid = releaseProcessPid(/** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath})), "v1", "web")
112
+ if (webPid === undefined) throw new Error("The active release did not report a web PID")
113
+ if (owner.pid === undefined) throw new Error("The daemon child did not report a PID")
114
+
115
+ owner.kill("SIGTERM")
116
+ await waitForProcessExit(owner.pid, 5000)
117
+ expect(isProcessRunning(webPid)).toBe(true)
118
+ } finally {
119
+ await sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath}).catch(() => undefined)
120
+ await killChild(owner)
121
+ await stopFixtureGuardian(fixture.statePath)
122
+ await fs.rm(fixture.root, {force: true, recursive: true})
123
+ }
124
+ })
125
+
102
126
  test("exact bootstrap restores the committed generation after external owner retirement", async () => {
103
127
  const fixture = await createFixture()
104
128
  const config = normalizeConfig(fixture.config, fixture.configPath)
@@ -720,7 +744,10 @@ test("guardian recovery becomes ready before replaying a gated generation hook",
720
744
  await fs.writeFile(retirementGatePath, "release retirement\n")
721
745
  await waitForProcessExit(recoveredDaemonPid, 5000)
722
746
  await waitForProcessExit(guardianPid, 5000)
723
- expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2", "retire:v2", "retire:v1"])
747
+ // The old generation is quiesced only after traffic switched to the candidate, so its quiet
748
+ // hook runs first; the candidate's own quiet hook can only run at shutdown. This is the order
749
+ // the same file asserts at lines 177 and 313.
750
+ expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2", "retire:v1", "retire:v2"])
724
751
  } finally {
725
752
  await fs.writeFile(retirementGatePath, "release retirement\n").catch(() => undefined)
726
753
  await sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath}).catch(() => undefined)
@@ -2171,7 +2171,7 @@ async function prepareCandidatePackage(destination, options = {}) {
2171
2171
  if (dropCommitResponse) {
2172
2172
  const clientPath = path.join(destination, "src", "control-client.js")
2173
2173
  const source = await fs.readFile(clientPath, "utf8")
2174
- const marker = "export async function sendControlCommand({command, path}) {\n"
2174
+ const marker = "export async function sendControlCommand({command, path, timeoutMs = DEFAULT_CONTROL_COMMAND_TIMEOUT_MS}) {\n"
2175
2175
  const injected = `${marker} if (command.command === "commit-owner-replacement") {\n return await new Promise((resolve, reject) => {\n const socket = net.createConnection(path)\n socket.once("error", reject)\n socket.once("data", () => { socket.destroy(); reject(new Error("injected lost commit response")) })\n socket.once("connect", () => socket.write(\`${"${JSON.stringify(command)}"}\\n\`))\n })\n }\n\n`
2176
2176
  const sessionMarker = " this.socket.write(`${JSON.stringify(command)}\\n`)\n return await response\n"
2177
2177
  const sessionInjection = " this.socket.write(`${JSON.stringify(command)}\\n`)\n const result = await response\n if (command.command === \"commit-owner-replacement\") throw new Error(\"injected lost commit response\")\n return result\n"
@@ -556,4 +556,43 @@ test("the retirement transition brackets its own wait and reports a residue it c
556
556
  fs.rmSync(procPath, {force: true, recursive: true})
557
557
  }
558
558
  })
559
+
560
+ test("retirement stores the refreshed definition identity it applied to the live generation", async () => {
561
+ const config = normalizeConfig({
562
+ application: "demo",
563
+ control: {path: "/tmp/rollbridge-release-group.sock"},
564
+ processes: [
565
+ {command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}},
566
+ {command: "run jobs", deployStrategy: "handoff", id: "jobs", policy: "service", port: {from: 0, to: 1}}
567
+ ],
568
+ proxy: {host: "127.0.0.1", port: 0}
569
+ })
570
+ const release = new ReleaseGroup({config, logger: () => {}, procPath: "/proc", releaseId: "v1", releasePath: "/tmp/rel", revision: "v1", shouldStart: () => true})
571
+ const generationConfig = release.config
572
+ /** @type {Array<import("../src/managed-process.js").ManagedProcessDefinition>} */
573
+ const refreshed = []
574
+
575
+ // eslint-disable-next-line local/no-unknown-jsdoc-type
576
+ release.processes.set("jobs", /** @type {import("../src/managed-process.js").default} */ (/** @type {unknown} */ ({
577
+ id: "jobs:v1",
578
+ quiesceStrict: async () => {},
579
+ status: () => ({pid: 4242, state: "running"}),
580
+ updateDefinition: (/** @type {import("../src/managed-process.js").ManagedProcessDefinition} */ definition) => { refreshed.push(definition) }
581
+ })))
582
+
583
+ const successor = {...release.config, processes: [release.config.processes[0], {...release.config.processes[1], env: {RELEASE_CONFIG_AUTHORITY: "v2"}}]}
584
+
585
+ await release.beginRetirement(successor)
586
+
587
+ // The live process is quiesced with the successor's definition, so the guardian records the
588
+ // successor's provenance for it. The generation's stored config must move with that definition:
589
+ // the durable owner state publishes it as the config recovery rebuilds each live process from, and
590
+ // a retained pre-refresh config makes the next recovery re-register the live process with the
591
+ // stale provenance and abort (STOE-2040).
592
+ expect(refreshed.map((definition) => definition.env?.RELEASE_CONFIG_AUTHORITY)).toEqual(["v2"])
593
+ expect(release.handoffServiceIds.has("jobs")).toBe(true)
594
+ expect(release.config).toBe(successor)
595
+ expect(release.config).not.toBe(generationConfig)
596
+ expect(release.state).toBe("draining")
597
+ })
559
598
  })