rollbridge 0.1.37 → 0.1.39

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/docs/cli.md CHANGED
@@ -64,6 +64,16 @@ identity. Rollbridge does not calculate or interpret the digest.
64
64
  With no release options, daemon behavior is unchanged: it starts listener-only
65
65
  and waits for control-socket deployments.
66
66
 
67
+ If an external owner retirement has already journaled a committed generation but
68
+ cleared its active role, only a foreground bootstrap with that exact release id,
69
+ path, revision, and config authority may restore it. Rollbridge waits for the
70
+ retiring candidate processes to stop, journals `restoring_committed`, reconnects
71
+ their existing guardian registrations, restarts that candidate, health-checks it,
72
+ and restores its generation activation before completing singletons and exposing
73
+ control. A later exact bootstrap resumes the journaled restart without duplicating
74
+ processes. A mismatched tuple or a candidate that is still retiring fails closed
75
+ without stopping other retained generations.
76
+
67
77
  `--takeover-owner` requires the complete bootstrap tuple. It bootstraps and
68
78
  health-checks the replacement before sending the current daemon the private
69
79
  retirement command. The current `performOwnerRetirement` path quiesces every
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "rollbridge",
3
- "version": "0.1.37",
3
+ "version": "0.1.39",
4
4
  "description": "Zero-downtime process supervisor and local traffic switcher for deploy-managed apps.",
5
5
  "keywords": [
6
6
  "deploy",
package/src/cli.js CHANGED
@@ -102,8 +102,13 @@ export async function runCli(argv) {
102
102
 
103
103
  daemon.logger("bootstrap activation failed", {releaseId: bootstrap.releaseId, status: "error", ...errorLogData(failure)})
104
104
 
105
- if (config.ownerRecovery && daemon.activeRelease) {
106
- await daemon.exposeControl()
105
+ if (config.ownerRecovery && daemon.releases.size > 0) {
106
+ if (daemon.activeRelease) {
107
+ await daemon.exposeControl()
108
+ return
109
+ }
110
+ await daemon.abandonOwnerRecoveryAttempt()
111
+ process.exitCode = 1
107
112
  return
108
113
  }
109
114
 
package/src/daemon.js CHANGED
@@ -23,7 +23,7 @@ const STATE_PERSIST_INTERVAL_MS = 5000
23
23
  * @typedef {{attestation?: string, releaseId: string, releasePath: string, revision: string}} BootstrapIdentity
24
24
  * @typedef {{id: string, process: import("./managed-process.js").ManagedProcessStatus}} ProcessStatus
25
25
  * @typedef {{disruptive: true, mode: "legacy-first-upgrade", reason: string}} OwnerTransition
26
- * @typedef {"candidate_ready" | "retiring_previous" | "previous_retired" | "activating_candidate" | "committed_pending" | "committed"} GenerationTransitionPhase
26
+ * @typedef {"candidate_ready" | "retiring_previous" | "previous_retired" | "activating_candidate" | "committed_pending" | "committed" | "restoring_committed"} GenerationTransitionPhase
27
27
  * @typedef {{candidateReleaseId: string, candidateReleasePath: string, candidateRevision: string, configDigest: string, error?: string, phase: GenerationTransitionPhase, previousReleaseId: string | null, startedAt: string, updatedAt: string}} GenerationTransition
28
28
  * @typedef {{activeReleaseId: string | null, application: string, bootstrap: BootstrapIdentity | undefined, control: import("./config.js").ControlConfig, daemonPid: number, daemonRuntime: import("./daemon-runtime.js").DaemonRuntimeIdentity | undefined, generationTransition?: GenerationTransition, ownerRecovery: {configDigest: string} | undefined, ownerTransition?: OwnerTransition, orphans: {id: string, pid: number, releaseId: string | null}[], proxy: {host: string, port: number | undefined, upstreamHost: string}, releaseReferences: {releaseId: string, releasePath: string}[], releases: import("./release-group.js").ReleaseStatus[], services: ProcessStatus[], singletons: ProcessStatus[]}} DaemonStatus
29
29
  * @typedef {{configDigest: string, format: number, guardian: {pid?: number, socketPath: string, token: string}, reconnectGraceMs: number}} OwnerRecoveryMetadata
@@ -228,7 +228,7 @@ export default class RollbridgeDaemon {
228
228
  }
229
229
  this.generationTransition = snapshot.generationTransition ? {...snapshot.generationTransition} : undefined
230
230
  if (snapshot.activeReleaseId === null && snapshot.releases.length === 0) return
231
- this.bootstrap = snapshot.bootstrap ? {...snapshot.bootstrap} : undefined
231
+ if (!this.bootstrap) this.bootstrap = snapshot.bootstrap ? {...snapshot.bootstrap} : undefined
232
232
  this.ownerTransition = snapshot.ownerTransition ? {...snapshot.ownerTransition} : undefined
233
233
  const singletonOwnerReleaseIds = new Set(Object.values(singletonReleaseIds))
234
234
 
@@ -255,9 +255,10 @@ export default class RollbridgeDaemon {
255
255
  }
256
256
 
257
257
  if (snapshot.activeReleaseId !== null && !this.activeRelease) throw new Error(`Owner recovery state does not contain active release ${snapshot.activeReleaseId}.`)
258
- const definitionRelease = this.activeRelease || [...this.releases.values()].at(-1)
258
+ const committedBootstrapRelease = this.committedBootstrapRelease()
259
+ const definitionRelease = this.activeRelease || committedBootstrapRelease || [...this.releases.values()].at(-1)
259
260
  if (!definitionRelease) throw new Error("Owner recovery state has no release definition for owned processes.")
260
- if (!this.activeRelease && snapshot.singletons.length > 0) throw new Error("Owner recovery state has release-owned singletons without an active release identity.")
261
+ if (!this.activeRelease && !committedBootstrapRelease && snapshot.singletons.length > 0) throw new Error("Owner recovery state has release-owned singletons without an active release identity.")
261
262
  for (const serviceStatus of snapshot.services) {
262
263
  const processConfig = config.processes.find((candidate) => candidate.id === serviceStatus.id && candidate.policy === "service" && candidate.deployStrategy !== "handoff")
263
264
 
@@ -340,7 +341,7 @@ export default class RollbridgeDaemon {
340
341
  if (!transfer?.config || !transfer.snapshot) throw new Error("Committed owner published incomplete replacement state")
341
342
  const registeredProcesses = new Map((await this.guardian.inventory()).map(({key, provenance}) => [key, provenance]))
342
343
 
343
- reservedProcessKey = legacyBridge ? undefined : ownerSnapshotProcessKeys(transfer.snapshot, transfer.singletonReleaseIds)
344
+ reservedProcessKey = legacyBridge ? undefined : reconstructableOwnerSnapshotProcessKeys(transfer.snapshot, transfer.singletonReleaseIds)
344
345
  .find((key) => registeredProcesses.has(key))
345
346
  if (reservedProcessKey) this.guardian.reserveProcessRecovery(reservedProcessKey, /** @type {string} */ (registeredProcesses.get(reservedProcessKey)))
346
347
  await this.restoreOwnerState(transfer.snapshot, {config: transfer.config, releaseConfigs: transfer.releaseConfigs, resumeDrains: false, singletonReleaseIds: transfer.singletonReleaseIds, synchronizeLifecycleRoles: false})
@@ -547,13 +548,14 @@ export default class RollbridgeDaemon {
547
548
  if (!releaseId || !connections || typeof connections !== "object" || Array.isArray(connections)) {
548
549
  throw new Error("Incumbent listener sent invalid connection state")
549
550
  }
550
- const release = this.releases.get(releaseId)
551
-
552
- if (!release) throw new Error(`Incumbent listener reported unknown release ${releaseId}`)
553
- release.setTransferredConnections({
551
+ const transferredConnections = {
554
552
  http: requiredNonNegativeInteger(connections.http, "connections.http"),
555
553
  websocket: requiredNonNegativeInteger(connections.websocket, "connections.websocket")
556
- })
554
+ }
555
+ const release = this.releases.get(releaseId)
556
+
557
+ if (!release && (transferredConnections.http > 0 || transferredConnections.websocket > 0)) throw new Error(`Incumbent listener reported unknown release ${releaseId}`)
558
+ if (release) release.setTransferredConnections(transferredConnections)
557
559
  if (this.incumbentListenerControl === session && ![...this.releases.values()].some((candidate) => candidate.hasTransferredConnections())) {
558
560
  this.incumbentListenerControl = undefined
559
561
  session.close()
@@ -1241,6 +1243,13 @@ export default class RollbridgeDaemon {
1241
1243
  this.assertExactGenerationTransition(transition, {config: nextConfig, releaseId: newReleaseId, releasePath, revision: revision || newReleaseId})
1242
1244
  return {activeReleaseId: newReleaseId, previousReleaseId: transition.previousReleaseId}
1243
1245
  }
1246
+ if (transition?.phase === "committed" && !this.activeRelease && this.bootstrap && this.releases.get(transition.candidateReleaseId)?.state === "draining") {
1247
+ this.assertExactGenerationTransition(transition, {config: nextConfig, releaseId: newReleaseId, releasePath, revision: revision || newReleaseId})
1248
+ this.assertCommittedBootstrapRecoveryReady()
1249
+ this.config = nextConfig
1250
+ await this.updateGenerationTransition("restoring_committed")
1251
+ return await this.resumeGenerationTransition()
1252
+ }
1244
1253
  const release = new ReleaseGroup({
1245
1254
  config: nextConfig,
1246
1255
  logger: this.logger,
@@ -1386,6 +1395,16 @@ export default class RollbridgeDaemon {
1386
1395
  this.logger("traffic switched", {previousReleaseId: previousRelease?.releaseId ?? null, releaseId: release.releaseId})
1387
1396
  }
1388
1397
 
1398
+ if (transition.phase === "restoring_committed") {
1399
+ await this.resumeCommittedBootstrapGeneration(release)
1400
+ release.activate()
1401
+ this.activeRelease = release
1402
+ transition.phase = "committed_pending"
1403
+ transition.error = undefined
1404
+ transition.updatedAt = new Date().toISOString()
1405
+ this.logger("committed bootstrap generation restored", {releaseId: release.releaseId})
1406
+ }
1407
+
1389
1408
  if (transition.phase === "committed_pending") {
1390
1409
  await this.checkpointGenerationTransition()
1391
1410
  this.refreshServiceDefinitions(release)
@@ -1419,6 +1438,82 @@ export default class RollbridgeDaemon {
1419
1438
  }
1420
1439
  }
1421
1440
 
1441
+ /**
1442
+ * Returns the retained candidate only when the foreground bootstrap exactly proves the
1443
+ * committed transition that an external owner retirement left without an active role.
1444
+ * @returns {ReleaseGroup | undefined} Exact committed bootstrap candidate.
1445
+ */
1446
+ committedBootstrapRelease() {
1447
+ const bootstrap = this.bootstrap
1448
+ const transition = this.generationTransition
1449
+
1450
+ if (!bootstrap || !transition || (transition.phase !== "committed" && transition.phase !== "restoring_committed") || transition.candidateReleaseId !== bootstrap.releaseId || transition.candidateReleasePath !== bootstrap.releasePath || transition.candidateRevision !== bootstrap.revision || transition.configDigest !== ownerConfigDigest(this.config)) return undefined
1451
+ const release = this.releases.get(bootstrap.releaseId)
1452
+
1453
+ if (!release || release.releasePath !== bootstrap.releasePath || release.revision !== bootstrap.revision) return undefined
1454
+ return release
1455
+ }
1456
+
1457
+ /**
1458
+ * Fails closed before journaling recovery unless external retirement has fully stopped
1459
+ * the exact candidate and daemon-owned processes.
1460
+ * @returns {void}
1461
+ */
1462
+ assertCommittedBootstrapRecoveryReady() {
1463
+ const release = this.committedBootstrapRelease()
1464
+
1465
+ if (!release) throw new Error("Committed generation has no exact foreground bootstrap recovery proof")
1466
+ release.assertCommittedGenerationStopped()
1467
+ const ownedProcesses = [
1468
+ ...this.services.values(),
1469
+ ...this.singletons.values()
1470
+ ]
1471
+ const stillRetiring = ownedProcesses.find((processInstance) => {
1472
+ const {pid, state} = processInstance.status()
1473
+ return pid !== undefined || (state !== "stopped" && state !== "failed")
1474
+ })
1475
+
1476
+ if (stillRetiring) throw new Error(`Committed generation daemon process ${stillRetiring.id} is still retiring; exact bootstrap recovery will retry after it stops`)
1477
+ for (const [singletonId, singletonReleaseId] of this.singletonReleaseIds) {
1478
+ if (singletonReleaseId !== release.releaseId) throw new Error(`Committed generation singleton ${singletonId} belongs to retained release ${singletonReleaseId}`)
1479
+ }
1480
+ }
1481
+
1482
+ /**
1483
+ * Resumes a durably journaled committed-candidate restart. Running processes are
1484
+ * necessarily owned by this exact recovery phase; stopped processes are restarted.
1485
+ * Singleton completion remains in the established committed_pending phase.
1486
+ * @param {ReleaseGroup} release - Exact committed candidate.
1487
+ * @returns {Promise<void>}
1488
+ */
1489
+ async resumeCommittedBootstrapGeneration(release) {
1490
+ if (this.generationTransition?.phase !== "restoring_committed" || release !== this.committedBootstrapRelease()) {
1491
+ throw new Error("Committed bootstrap recovery is not durably journaled for this exact candidate")
1492
+ }
1493
+ const resumableStates = new Set(["failed", "running", "stopped"])
1494
+ const invalidProcess = [...this.services.values(), ...this.singletons.values()].find((processInstance) => {
1495
+ const {pid, state} = processInstance.status()
1496
+ return !resumableStates.has(state) || (state === "running") !== (pid !== undefined)
1497
+ })
1498
+
1499
+ if (invalidProcess) throw new Error(`Committed bootstrap recovery found daemon process ${invalidProcess.id} outside its journaled restart states`)
1500
+ release.assertCommittedGenerationRecoverable()
1501
+
1502
+ try {
1503
+ for (const processInstance of this.services.values()) {
1504
+ await processInstance.start("deploy")
1505
+ }
1506
+ await release.restartCommittedGeneration()
1507
+ await release.activateGeneration()
1508
+ } catch (error) {
1509
+ await Promise.allSettled([
1510
+ release.abortCommittedGenerationRestart(),
1511
+ ...[...this.services.values()].map((processInstance) => processInstance.stop())
1512
+ ])
1513
+ throw error
1514
+ }
1515
+ }
1516
+
1422
1517
  /** @param {GenerationTransitionPhase} phase - Durable phase to enter. */
1423
1518
  async updateGenerationTransition(phase) {
1424
1519
  if (!this.generationTransition) throw new Error("No release generation transition to update")
@@ -1454,6 +1549,7 @@ export default class RollbridgeDaemon {
1454
1549
  shouldResumeDrain(release) {
1455
1550
  const transition = this.generationTransition
1456
1551
 
1552
+ if (release === this.committedBootstrapRelease()) return false
1457
1553
  return !transition || transition.phase === "committed_pending" || transition.phase === "committed" || transition.previousReleaseId !== release.releaseId
1458
1554
  }
1459
1555
 
@@ -2121,6 +2217,27 @@ function ownerSnapshotProcessKeys(snapshot, singletonReleaseIds = {}) {
2121
2217
  return keys
2122
2218
  }
2123
2219
 
2220
+ /**
2221
+ * Selects only committed guardian registrations that restoreOwnerState will reconstruct.
2222
+ * @param {OwnerRecoverySnapshot} snapshot - Serialized owner process snapshot.
2223
+ * @param {Record<string, string>} [singletonReleaseIds] - Exact singleton owner releases.
2224
+ * @returns {string[]} Exact reconstructable guardian registration keys.
2225
+ */
2226
+ function reconstructableOwnerSnapshotProcessKeys(snapshot, singletonReleaseIds = {}) {
2227
+ const singletonOwnerReleaseIds = new Set(Object.values(singletonReleaseIds))
2228
+ const releaseProcessKeys = new Set()
2229
+
2230
+ for (const release of snapshot.releases) {
2231
+ const transitionCandidate = snapshot.generationTransition?.candidateReleaseId === release.releaseId && snapshot.generationTransition.phase !== "committed"
2232
+ const singletonOwner = singletonOwnerReleaseIds.has(release.releaseId)
2233
+
2234
+ if (release.state !== "active" && release.state !== "draining" && !transitionCandidate && !singletonOwner) continue
2235
+ for (const processStatus of release.processes) releaseProcessKeys.add(`release:${release.releaseId}:${processStatus.id}`)
2236
+ }
2237
+ return ownerSnapshotProcessKeys(snapshot, singletonReleaseIds)
2238
+ .filter((key) => !key.startsWith("release:") || releaseProcessKeys.has(key))
2239
+ }
2240
+
2124
2241
  /**
2125
2242
  * Canonicalizes the authenticated guardian inventory without relying on map insertion order.
2126
2243
  * @param {{key: string, provenance: string}[]} inventory - Guardian inventory response.
@@ -276,6 +276,61 @@ export default class ReleaseGroup extends EventEmitter {
276
276
  await instance.process.activateStrict()
277
277
  }
278
278
 
279
+ /**
280
+ * Restarts only the exact processes reconstructed for a committed generation.
281
+ * The caller must prove the durable transition identity before using this path.
282
+ */
283
+ async restartCommittedGeneration() {
284
+ this.assertCommittedGenerationRecoverable()
285
+ this.state = "starting"
286
+ try {
287
+ for (const processConfig of this.releaseProcessStartOrder()) {
288
+ const instances = this.getProcesses(processConfig.id)
289
+
290
+ if (instances.length !== processConfig.replicas) throw new Error(`Committed generation ${this.releaseId} is missing process ${processConfig.id}`)
291
+ for (const {process} of instances) await process.start("deploy", processConfig.lifecycle.activateCommand ? "candidate" : undefined)
292
+
293
+ if (processConfig.policy === "proxied" && processConfig.port && processConfig.health) {
294
+ await waitForHealth({
295
+ health: processConfig.health,
296
+ host: this.config.proxy.upstreamHost,
297
+ port: this.ports[processConfig.id]
298
+ })
299
+ }
300
+ }
301
+ } catch (error) {
302
+ await this.abortCommittedGenerationRestart()
303
+ throw error
304
+ }
305
+ }
306
+
307
+ /** Fails closed unless every exact candidate process has finished external retirement. */
308
+ assertCommittedGenerationStopped() {
309
+ if (this.state !== "draining") throw new Error(`Committed generation ${this.releaseId} is not retained as draining`)
310
+ const statuses = [...this.processes.values()].map((processInstance) => processInstance.status())
311
+
312
+ if (statuses.some(({pid, state}) => pid !== undefined || (state !== "stopped" && state !== "failed"))) {
313
+ throw new Error(`Committed generation ${this.releaseId} is still retiring; exact bootstrap recovery will retry after its processes stop`)
314
+ }
315
+ }
316
+
317
+ /** Accepts only process states produced after the exact recovery phase was journaled. */
318
+ assertCommittedGenerationRecoverable() {
319
+ if (this.state !== "draining" && this.state !== "starting") throw new Error(`Committed generation ${this.releaseId} is not retained in its journaled recovery state`)
320
+ const statuses = [...this.processes.values()].map((processInstance) => processInstance.status())
321
+ const resumableStates = new Set(["failed", "running", "stopped"])
322
+
323
+ if (statuses.some(({pid, state}) => !resumableStates.has(state) || (state === "running") !== (pid !== undefined))) {
324
+ throw new Error(`Committed generation ${this.releaseId} has a process outside its journaled restart states`)
325
+ }
326
+ }
327
+
328
+ /** Stops only a failed committed-bootstrap restart while retaining its exact identity. */
329
+ async abortCommittedGenerationRestart() {
330
+ await Promise.allSettled([...this.processes.values()].map((processInstance) => processInstance.stop()))
331
+ this.state = "draining"
332
+ }
333
+
279
334
  /** @returns {Promise<void>} Allocates all configured per-process ports. */
280
335
  async allocatePorts() {
281
336
  if (this.portsAllocated) return
@@ -95,6 +95,259 @@ test("external owner retirement releases guardian authority without losing its g
95
95
  }
96
96
  })
97
97
 
98
+ test("exact bootstrap restores the committed generation after external owner retirement", async () => {
99
+ const fixture = await createFixture()
100
+ const config = normalizeConfig(fixture.config, fixture.configPath)
101
+ const retired = new RollbridgeDaemon({config, configPath: fixture.configPath, logger: () => {}})
102
+ /** @type {RollbridgeDaemon | undefined} */
103
+ let recovered
104
+ const v1Path = await prepareRelease(fixture.root, "v1")
105
+ const v2Path = await prepareRelease(fixture.root, "v2")
106
+
107
+ try {
108
+ await retired.start()
109
+ await retired.deploy({releaseId: "v1", releasePath: v1Path, revision: "v1"})
110
+ await retired.deploy({releaseId: "v2", releasePath: v2Path, revision: "v2"})
111
+ const committed = retired.status()
112
+ const v1WorkerPid = releaseProcessPid(committed, "v1", "worker")
113
+ const retiredPids = [
114
+ ...(committed.releases.find(({releaseId}) => releaseId === "v2")?.processes.map(({pid}) => pid) || []),
115
+ ...committed.services.map(({process}) => process.pid),
116
+ ...committed.singletons.map(({process}) => process.pid)
117
+ ].filter((pid) => typeof pid === "number")
118
+
119
+ assert.equal(committed.activeReleaseId, "v2")
120
+ assert.equal(committed.generationTransition?.phase, "committed")
121
+ await retired.retireOwner({attestation: `sha256:${"a".repeat(64)}`})
122
+ await fs.writeFile(path.join(v2Path, "worker.fifo"), "drained\n")
123
+ await Promise.all(retiredPids.map((pid) => waitForProcessExit(pid, 3000)))
124
+
125
+ recovered = new RollbridgeDaemon({
126
+ bootstrap: {releaseId: "v2", releasePath: v2Path, revision: "v2"},
127
+ config,
128
+ configPath: fixture.configPath,
129
+ logger: () => {}
130
+ })
131
+ await recovered.start({exposeControl: false})
132
+ const recoveryOrder = /** @type {string[]} */ ([])
133
+ const recoveredCandidate = recovered.releases.get("v2")
134
+ const recoveredService = recovered.services.get("beacon")
135
+
136
+ assert.ok(recoveredCandidate && recoveredService && recovered.singletons.get("singleton"))
137
+ const activateGeneration = recoveredCandidate.activateGeneration.bind(recoveredCandidate)
138
+ const startService = recoveredService.start.bind(recoveredService)
139
+ const replaceSingletons = recovered.replaceSingletons.bind(recovered)
140
+
141
+ recoveredCandidate.activateGeneration = async () => {
142
+ recoveryOrder.push("activate")
143
+ await activateGeneration()
144
+ }
145
+ recoveredService.start = async (...args) => {
146
+ assert.equal(recovered?.generationTransition?.phase, "restoring_committed")
147
+ recoveryOrder.push("service")
148
+ await startService(...args)
149
+ }
150
+ recovered.replaceSingletons = async (...args) => {
151
+ recoveryOrder.push("singleton")
152
+ await replaceSingletons(...args)
153
+ }
154
+ await recovered.deploy({releaseId: "v2", releasePath: v2Path, revision: "v2"})
155
+ const active = recovered.status()
156
+
157
+ assert.equal(active.activeReleaseId, "v2")
158
+ assert.equal(active.generationTransition?.phase, "committed")
159
+ assert.equal(active.releases.find(({releaseId}) => releaseId === "v1")?.state, "draining")
160
+ assert.equal(releaseProcessPid(active, "v1", "worker"), v1WorkerPid)
161
+ assert.equal(isAlive(v1WorkerPid), true, "the retained previous generation must keep draining")
162
+ assert.equal(active.releases.find(({releaseId}) => releaseId === "v2")?.state, "active")
163
+ assert.ok(active.releases.find(({releaseId}) => releaseId === "v2")?.processes.every(({pid, state}) => typeof pid === "number" && state === "running"))
164
+ assert.ok(active.services.every(({process}) => typeof process.pid === "number" && process.state === "running"))
165
+ assert.ok(active.singletons.every(({process}) => typeof process.pid === "number" && process.state === "running"))
166
+ assert.deepEqual(recoveryOrder, ["service", "activate", "singleton"], "candidate activation must precede post-commit singleton completion")
167
+ assert.deepEqual(await lifecycleEvents(fixture.lifecycleLogPath), ["activate:v1", "retire:v1", "activate:v2", "retire:v2", "activate:v2"])
168
+ } finally {
169
+ if (recovered) {
170
+ const activeRecovery = recovered.status().activeReleaseId === "v2"
171
+ const shutdown = recovered.shutdown()
172
+
173
+ await fs.writeFile(path.join(v1Path, "worker.fifo"), "drained\n").catch(() => undefined)
174
+ if (activeRecovery) await fs.writeFile(path.join(v2Path, "worker.fifo"), "drained\n").catch(() => undefined)
175
+ await shutdown.catch(() => undefined)
176
+ }
177
+ retired.guardian?.disconnect()
178
+ await stopFixtureGuardian(fixture.statePath)
179
+ await fs.rm(fixture.root, {force: true, recursive: true})
180
+ }
181
+ })
182
+
183
+ test("journaled committed bootstrap recovery resumes after a restart begins", async () => {
184
+ const fixture = await createFixture()
185
+ const config = normalizeConfig(fixture.config, fixture.configPath)
186
+ const retired = new RollbridgeDaemon({config, configPath: fixture.configPath, logger: () => {}})
187
+ /** @type {RollbridgeDaemon | undefined} */
188
+ let interrupted
189
+ /** @type {RollbridgeDaemon | undefined} */
190
+ let recovered
191
+ const v1Path = await prepareRelease(fixture.root, "v1")
192
+ const v2Path = await prepareRelease(fixture.root, "v2")
193
+
194
+ try {
195
+ await retired.start()
196
+ await retired.deploy({releaseId: "v1", releasePath: v1Path, revision: "v1"})
197
+ await retired.deploy({releaseId: "v2", releasePath: v2Path, revision: "v2"})
198
+ const committed = retired.status()
199
+ const v1WorkerPid = releaseProcessPid(committed, "v1", "worker")
200
+ const retiringPids = [
201
+ ...(committed.releases.find(({releaseId}) => releaseId === "v2")?.processes.map(({pid}) => pid) || []),
202
+ ...committed.services.map(({process}) => process.pid),
203
+ ...committed.singletons.map(({process}) => process.pid)
204
+ ].filter((pid) => typeof pid === "number")
205
+
206
+ await retired.retireOwner({attestation: `sha256:${"b".repeat(64)}`})
207
+ await fs.writeFile(path.join(v2Path, "worker.fifo"), "drained\n")
208
+ await Promise.all(retiringPids.map((pid) => waitForProcessExit(pid, 3000)))
209
+
210
+ const bootstrap = {releaseId: "v2", releasePath: v2Path, revision: "v2"}
211
+ interrupted = new RollbridgeDaemon({bootstrap, config, configPath: fixture.configPath, logger: () => {}})
212
+ await interrupted.start({exposeControl: false})
213
+ interrupted.assertCommittedBootstrapRecoveryReady()
214
+ await interrupted.updateGenerationTransition("restoring_committed")
215
+ const candidate = interrupted.releases.get("v2")
216
+
217
+ assert.ok(candidate)
218
+ for (const processInstance of interrupted.services.values()) await processInstance.start("deploy")
219
+ await candidate.restartCommittedGeneration()
220
+ await interrupted.checkpointGenerationTransition()
221
+ const restarted = interrupted.status()
222
+ const restartedCandidatePids = restarted.releases.find(({releaseId}) => releaseId === "v2")?.processes.map(({pid}) => pid)
223
+ const restartedServicePids = restarted.services.map(({process}) => process.pid)
224
+
225
+ assert.equal(restarted.generationTransition?.phase, "restoring_committed")
226
+ assert.ok(restartedCandidatePids?.every((pid) => typeof pid === "number"))
227
+ await interrupted.retireCommittedOwner(undefined)
228
+ interrupted.guardian?.disconnect()
229
+
230
+ recovered = new RollbridgeDaemon({bootstrap, config, configPath: fixture.configPath, logger: () => {}})
231
+ await recovered.start({exposeControl: false})
232
+ const active = recovered.status()
233
+
234
+ assert.equal(active.activeReleaseId, "v2")
235
+ assert.equal(active.generationTransition?.phase, "committed")
236
+ assert.deepEqual(active.releases.find(({releaseId}) => releaseId === "v2")?.processes.map(({pid}) => pid), restartedCandidatePids)
237
+ assert.deepEqual(active.services.map(({process}) => process.pid), restartedServicePids)
238
+ assert.ok(active.singletons.every(({process}) => typeof process.pid === "number" && process.state === "running"))
239
+ assert.equal(releaseProcessPid(active, "v1", "worker"), v1WorkerPid)
240
+ assert.equal(isAlive(v1WorkerPid), true)
241
+ assert.deepEqual(await lifecycleEvents(fixture.lifecycleLogPath), ["activate:v1", "retire:v1", "activate:v2", "retire:v2", "activate:v2"])
242
+ } finally {
243
+ if (recovered) {
244
+ const shutdown = recovered.shutdown()
245
+
246
+ await Promise.all([v1Path, v2Path].map((releasePath) => fs.writeFile(path.join(releasePath, "worker.fifo"), "drained\n").catch(() => undefined)))
247
+ await shutdown.catch(() => undefined)
248
+ }
249
+ interrupted?.guardian?.disconnect()
250
+ retired.guardian?.disconnect()
251
+ await stopFixtureGuardian(fixture.statePath)
252
+ await fs.rm(fixture.root, {force: true, recursive: true})
253
+ }
254
+ })
255
+
256
+ test("committed bootstrap tuple mismatches fail closed without singletons", async () => {
257
+ const fixture = await createFixture()
258
+ fixture.config.processes = /** @type {Record<string, import("../src/json.js").JsonValue>[]} */ (fixture.config.processes)
259
+ .filter((processConfig) => processConfig.id !== "singleton")
260
+ await writeConfig(fixture.configPath, fixture.config)
261
+ const config = normalizeConfig(fixture.config, fixture.configPath)
262
+ const retired = new RollbridgeDaemon({config, configPath: fixture.configPath, logger: () => {}})
263
+ /** @type {RollbridgeDaemon | undefined} */
264
+ let recovered
265
+ const v1Path = await prepareRelease(fixture.root, "v1")
266
+ const v2Path = await prepareRelease(fixture.root, "v2")
267
+ const wrongPath = await prepareRelease(fixture.root, "wrong")
268
+
269
+ try {
270
+ await retired.start()
271
+ await retired.deploy({releaseId: "v1", releasePath: v1Path, revision: "v1"})
272
+ await retired.deploy({releaseId: "v2", releasePath: v2Path, revision: "v2"})
273
+ const committed = retired.status()
274
+ const v1WorkerPid = releaseProcessPid(committed, "v1", "worker")
275
+ const retiringPids = [
276
+ ...(committed.releases.find(({releaseId}) => releaseId === "v2")?.processes.map(({pid}) => pid) || []),
277
+ ...committed.services.map(({process}) => process.pid)
278
+ ].filter((pid) => typeof pid === "number")
279
+
280
+ await retired.retireOwner({attestation: `sha256:${"c".repeat(64)}`})
281
+ await fs.writeFile(path.join(v2Path, "worker.fifo"), "drained\n")
282
+ await Promise.all(retiringPids.map((pid) => waitForProcessExit(pid, 3000)))
283
+ recovered = new RollbridgeDaemon({
284
+ bootstrap: {releaseId: "v2", releasePath: v2Path, revision: "v2"},
285
+ config,
286
+ configPath: fixture.configPath,
287
+ logger: () => {}
288
+ })
289
+ await recovered.start({exposeControl: false})
290
+ const owner = recovered
291
+
292
+ await assert.rejects(
293
+ () => owner.deploy({releaseId: "v2", releasePath: wrongPath, revision: "v2"}),
294
+ /only the exact same release, path, revision, and config authority/u
295
+ )
296
+ await assert.rejects(
297
+ () => owner.deploy({releaseId: "v2", releasePath: v2Path, revision: "wrong"}),
298
+ /only the exact same release, path, revision, and config authority/u
299
+ )
300
+ const changedConfig = structuredClone(fixture.config)
301
+ const jobs = /** @type {Record<string, import("../src/json.js").JsonValue>[]} */ (changedConfig.processes)
302
+ .find((processConfig) => processConfig.id === "jobs")
303
+
304
+ assert.ok(jobs)
305
+ jobs.env = {.../** @type {Record<string, import("../src/json.js").JsonValue>} */ (jobs.env), MISMATCHED_AUTHORITY: "true"}
306
+ await writeConfig(fixture.configPath, changedConfig)
307
+ await assert.rejects(
308
+ () => owner.deploy({releaseId: "v2", releasePath: v2Path, revision: "v2"}),
309
+ /only the exact same release, path, revision, and config authority/u
310
+ )
311
+ await writeConfig(fixture.configPath, fixture.config)
312
+ await assert.rejects(
313
+ () => owner.deploy({releaseId: "wrong", releasePath: wrongPath, revision: "wrong"}),
314
+ /only the exact same release, path, revision, and config authority/u
315
+ )
316
+ const preserved = owner.status()
317
+
318
+ assert.equal(preserved.activeReleaseId, null)
319
+ assert.equal(preserved.generationTransition?.candidateReleaseId, "v2")
320
+ assert.equal(preserved.generationTransition?.phase, "committed")
321
+ assert.equal(preserved.releases.find(({releaseId}) => releaseId === "v2")?.state, "draining")
322
+ assert.equal(releaseProcessPid(preserved, "v1", "worker"), v1WorkerPid)
323
+ assert.equal(isAlive(v1WorkerPid), true)
324
+ assert.equal(preserved.releases.some(({releaseId}) => releaseId === "wrong"), false)
325
+
326
+ await owner.deploy({releaseId: "v2", releasePath: v2Path, revision: "v2"})
327
+ await Promise.all([
328
+ owner.stopRelease("v2"),
329
+ fs.writeFile(path.join(v2Path, "worker.fifo"), "drained\n")
330
+ ])
331
+ const afterIntentionalStop = await owner.deploy({releaseId: "wrong", releasePath: wrongPath, revision: "wrong"})
332
+
333
+ assert.equal(afterIntentionalStop.activeReleaseId, "wrong")
334
+ assert.equal(owner.status().activeReleaseId, "wrong")
335
+ } finally {
336
+ if (recovered) {
337
+ const activeReleaseId = recovered.status().activeReleaseId
338
+ const shutdown = recovered.shutdown()
339
+
340
+ await fs.writeFile(path.join(v1Path, "worker.fifo"), "drained\n").catch(() => undefined)
341
+ if (activeReleaseId === "v2") await fs.writeFile(path.join(v2Path, "worker.fifo"), "drained\n").catch(() => undefined)
342
+ if (activeReleaseId === "wrong") await fs.writeFile(path.join(wrongPath, "worker.fifo"), "drained\n").catch(() => undefined)
343
+ await shutdown.catch(() => undefined)
344
+ }
345
+ retired.guardian?.disconnect()
346
+ await stopFixtureGuardian(fixture.statePath)
347
+ await fs.rm(fixture.root, {force: true, recursive: true})
348
+ }
349
+ })
350
+
98
351
  test("replacement owner reconstructs one active and two draining generations after abrupt daemon exit", async () => {
99
352
  const fixture = await createFixture()
100
353
  let owner = spawnDaemon(fixture.configPath)
@@ -10,7 +10,7 @@ import path from "node:path"
10
10
  import test from "node:test"
11
11
  import {fileURLToPath} from "node:url"
12
12
  import {normalizeConfig} from "../src/config.js"
13
- import {sendControlCommand} from "../src/control-client.js"
13
+ import {openControlSession, sendControlCommand} from "../src/control-client.js"
14
14
  import RollbridgeDaemon, {isLegacyGuardianPrepareDiagnostic} from "../src/daemon.js"
15
15
  import GuardianClient from "../src/guardian-client.js"
16
16
  import {findAvailablePort} from "../src/port-allocator.js"
@@ -1188,6 +1188,88 @@ test("owner replacement preserves committed generation metadata without firing l
1188
1188
  }
1189
1189
  })
1190
1190
 
1191
+ test("owner replacement excludes stopped retained releases from reserved process recovery", async () => {
1192
+ const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-owner-replacement-stopped-proof-"))
1193
+ const oldSocketPath = path.join(root, "old.sock")
1194
+ const newSocketPath = path.join(root, "new.sock")
1195
+ const statePath = path.join(root, "state.json")
1196
+ const configPath = path.join(root, "rollbridge.cjs")
1197
+ const v1Path = path.join(root, "v1")
1198
+ const v2Path = path.join(root, "v2")
1199
+ let owner
1200
+ let candidate
1201
+
1202
+ try {
1203
+ await Promise.all([fs.mkdir(v1Path), fs.mkdir(v2Path)])
1204
+ await Promise.all([makeFifo(path.join(v1Path, "worker.fifo")), makeFifo(path.join(v2Path, "worker.fifo"))])
1205
+ await writeConfig(configPath, config({controlPath: oldSocketPath, extraCompanion: false, statePath}))
1206
+ owner = spawn(process.execPath, [binPath, "daemon", "--config", configPath], {stdio: ["ignore", "pipe", "pipe"]})
1207
+ await waitForLog(owner, "control socket listening")
1208
+ await sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath: v1Path, revision: "v1"}, path: oldSocketPath})
1209
+ const stopped = sendControlCommand({command: {command: "stop", releaseId: "v1"}, path: oldSocketPath})
1210
+
1211
+ await fs.writeFile(path.join(v1Path, "worker.fifo"), "drained\n")
1212
+ await stopped
1213
+ await sendControlCommand({command: {command: "deploy", releaseId: "v2", releasePath: v2Path, revision: "v2"}, path: oldSocketPath})
1214
+ const before = await sendControlCommand({command: {command: "status"}, path: oldSocketPath})
1215
+ const activeWorkerPid = releaseProcessPid(before, "v2", "worker")
1216
+ const retained = /** @type {{releaseId: string, state: string}[]} */ (before.releases)
1217
+
1218
+ assert.equal(retained.find(({releaseId}) => releaseId === "v1")?.state, "stopped")
1219
+ const persisted = JSON.parse(await fs.readFile(statePath, "utf8"))
1220
+ const guardian = new GuardianClient(persisted.recovery.guardian)
1221
+
1222
+ await guardian.connect()
1223
+ assert.ok((await guardian.inventory()).some(({key}) => key === "release:v1:worker"), "stopped release registration remains in authenticated guardian inventory")
1224
+ guardian.disconnect()
1225
+ await writeConfig(configPath, config({controlPath: newSocketPath, extraCompanion: true, statePath}))
1226
+ candidate = spawn(process.execPath, [binPath, "daemon", "--config", configPath, "--replace-owner"], {stdio: ["ignore", "pipe", "pipe"]})
1227
+ const output = await collectUntilExitOrLog(candidate, "owner replacement committed")
1228
+
1229
+ assert.equal(output.message, "owner replacement committed", output.output)
1230
+ const recovered = await sendControlCommand({command: {command: "status"}, path: newSocketPath})
1231
+
1232
+ assert.equal(recovered.activeReleaseId, "v2")
1233
+ assert.equal(releaseProcessPid(recovered, "v2", "worker"), activeWorkerPid)
1234
+ assert.equal(/** @type {{releaseId: string}[]} */ (recovered.releases).some(({releaseId}) => releaseId === "v1"), false)
1235
+ const shutdown = sendControlCommand({command: {command: "shutdown"}, path: newSocketPath})
1236
+
1237
+ await fs.writeFile(path.join(v2Path, "worker.fifo"), "drained\n")
1238
+ await shutdown
1239
+ } finally {
1240
+ for (const child of [owner, candidate]) if (child && child.exitCode === null && child.signalCode === null) child.kill("SIGKILL")
1241
+ await stopGuardian(statePath)
1242
+ await fs.rm(root, {force: true, recursive: true})
1243
+ }
1244
+ })
1245
+
1246
+ test("pruned release connection completion closes the incumbent listener session", () => {
1247
+ const daemon = new RollbridgeDaemon({
1248
+ config: normalizeConfig(config({controlPath: "/unused/control.sock", extraCompanion: false, statePath: "/unused/state.json"})),
1249
+ logger: () => {}
1250
+ })
1251
+ let closeCount = 0
1252
+ const session = {close: () => { closeCount += 1 }}
1253
+ const controlSession = /** @type {Awaited<ReturnType<typeof openControlSession>>} */ (session)
1254
+
1255
+ daemon.releases = /** @type {Map<string, import("../src/release-group.js").default>} */ (new Map([
1256
+ ["active", /** @type {import("../src/release-group.js").default} */ ({hasTransferredConnections: () => false})]
1257
+ ]))
1258
+ daemon.incumbentListenerControl = controlSession
1259
+ daemon.handleIncumbentListenerEvent({connections: {http: 0, websocket: 0}, event: "owner-connection-state", releaseId: "pruned"}, session)
1260
+
1261
+ assert.equal(closeCount, 1)
1262
+ assert.equal(daemon.incumbentListenerControl, undefined)
1263
+
1264
+ daemon.incumbentListenerControl = controlSession
1265
+ assert.throws(
1266
+ () => daemon.handleIncumbentListenerEvent({connections: {http: 1, websocket: 0}, event: "owner-connection-state", releaseId: "pruned"}, session),
1267
+ /unknown release pruned/
1268
+ )
1269
+ assert.equal(closeCount, 1)
1270
+ assert.equal(daemon.incumbentListenerControl, session)
1271
+ })
1272
+
1191
1273
  test("owner replacement preserves a failed generation transition without retrying its hook", async () => {
1192
1274
  const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-owner-replacement-failed-generation-"))
1193
1275
  const oldSocketPath = path.join(root, "old.sock")