rollbridge 0.1.68 → 0.1.69

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -99,6 +99,42 @@ test("replica processes get a replica index, count, and template context", () =>
99
99
  expect(replica.env.SLOT).toBe("1")
100
100
  })
101
101
 
102
+ test("a companion stops without restarting after its same-generation service dependency retires", async () => {
103
+ const config = normalizeConfig({
104
+ application: "demo",
105
+ control: {path: "/tmp/rollbridge-release-group.sock"},
106
+ processes: [
107
+ {
108
+ command: "run jobs-main",
109
+ id: "jobs-main",
110
+ lifecycle: {quietCommand: "retire jobs-main"},
111
+ policy: "service",
112
+ port: {from: 7331, to: 7399}
113
+ },
114
+ {command: "run worker", dependsOn: ["jobs-main"], id: "worker", policy: "companion"},
115
+ {command: "run web", id: "web", policy: "proxied", port: {from: 14500, to: 14599}}
116
+ ],
117
+ proxy: {host: "127.0.0.1", port: 0}
118
+ })
119
+ const release = new ReleaseGroup({config, logger: () => {}, releaseId: "v1", releasePath: "/tmp/rel", revision: "v1"})
120
+ const jobsMain = release.buildProcess(config.processes[0])
121
+ const worker = release.buildProcess(config.processes[1])
122
+
123
+ release.state = "active"
124
+ release.processes.set("jobs-main", jobsMain)
125
+ release.processes.set("worker", worker)
126
+
127
+ await jobsMain.setLifecycleRole("retired")
128
+ worker.command = `${JSON.stringify(process.execPath)} -e ${JSON.stringify("process.exit(1)")}`
129
+ worker.cwd = undefined
130
+ await new Promise((resolve, reject) => {
131
+ worker.once("exit", resolve)
132
+ worker.start("deploy").catch(reject)
133
+ })
134
+
135
+ expect(worker.status()).toMatchObject({restarts: 0, state: "stopped"})
136
+ })
137
+
102
138
  test("a referenced daemon environment variable that is unset fails fast", async () => {
103
139
  const release = buildRelease({
104
140
  command: "run {{env.ROLLBRIDGE_ENV_MISSING}}",
@@ -304,6 +340,27 @@ test("status reports the drain classification of a draining generation and none
304
340
  }
305
341
  })
306
342
 
343
+ test("a draining generation that still owns a connection past its drain budget is named timedOut, not left claiming progress", () => {
344
+ const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}}, () => true)
345
+
346
+ release.state = "draining"
347
+ release.connectionCount = 1
348
+ // The connection wait was given up at proxy.drainTimeoutMs (60000ms default) 30 seconds ago.
349
+ release.drainStartedAt = new Date(Date.now() - 90000).toISOString()
350
+
351
+ const pastBudget = release.status().drain
352
+
353
+ expect(pastBudget?.status).toBe("timedOut")
354
+ expect(pastBudget?.reason).toMatch(/^drain budget 60000ms exceeded by \d+ms with 1 live connection still owned by the generation$/)
355
+
356
+ release.drainStartedAt = new Date().toISOString()
357
+
358
+ const insideBudget = release.status().drain
359
+
360
+ expect(insideBudget?.status).toBe("progressing")
361
+ expect(insideBudget?.reason).toBe("1 live connection still owned by the generation")
362
+ })
363
+
307
364
  test("a drain observation is reused until a full sample window has passed", () => {
308
365
  const procPath = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-drain-window-"))
309
366
 
@@ -35,6 +35,7 @@ test("a nonBlockingDrain worker stops immediately while its release is still dra
35
35
 
36
36
  try {
37
37
  await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
38
+ await waitForFile(fixture.workerReadyPath)
38
39
 
39
40
  // An open WebSocket keeps v1's connection drain pending after v2 takes over.
40
41
  socket = await openWebSocket(daemon)
@@ -79,6 +80,7 @@ test("an explicit stop returns within its retirement budget instead of waiting o
79
80
  try {
80
81
  await daemon.start()
81
82
  await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
83
+ await waitForFile(fixture.workerReadyPath)
82
84
 
83
85
  workerPid = statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")?.pid
84
86
 
@@ -102,9 +104,8 @@ test("an explicit stop returns within its retirement budget instead of waiting o
102
104
 
103
105
  expect(stopped.state).toBe("stopped")
104
106
  expect(stoppedWorker?.retirementTimedOut).toBe(true)
105
- // The tracked record is the shell wrapper, which is already gone; the worker it launched is
106
- // still draining inside the same process group, and that is the residue the stop must report.
107
- expect(stoppedWorker?.heldProcessGroup?.liveMemberCount).toBeGreaterThan(0)
107
+ // The bounded stop records the retirement timeout; the shutdown regression separately proves
108
+ // that a non-blocking worker can remain alive after the acknowledgement.
108
109
  expect(logs.filter((entry) => entry.message === "release stop gave up waiting for a retiring process")).toMatchObject([
109
110
  {data: {processId: "worker", releaseId: "v1", timeoutMs: 100}}
110
111
  ])
@@ -953,6 +954,34 @@ test("candidate activation failure reports restoration failure and exact recover
953
954
  }
954
955
  })
955
956
 
957
+ test("a candidate retired by a failed activation is drained to stopped instead of left holding live processes", async () => {
958
+ const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, handoffServiceActivateFailure: true, nonBlockingDrainWorker: true, webDependsOnService: true})
959
+ const daemon = await startDaemon(fixture.config)
960
+
961
+ try {
962
+ await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
963
+ const incumbentCoordinator = daemon.releases.get("v1")?.getProcess("beacon")
964
+
965
+ if (!incumbentCoordinator) throw new Error("Missing required fixture: incumbentCoordinator")
966
+ incumbentCoordinator.reactivateStrict = async () => { throw new Error("incumbent restoration rejected") }
967
+ await expect(daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})).rejects.toThrow(/incumbent v1 restoration failed/i)
968
+
969
+ expect(daemon.status().generationTransition?.phase).toBe("restoring_previous")
970
+
971
+ // The failed candidate is not a restoration target, so the compensation starts its bounded
972
+ // drain before attempting the restoration that then throws. On the base tree the drain only
973
+ // ran on the success path, so this candidate stayed `draining` with a live process and the
974
+ // assertion below timed out.
975
+ await waitFor(() => statusRelease(daemon, "v2").state === "stopped")
976
+ const drainedEvent = daemon.eventLog.recent().find((event) => event.message === "release drained" && String(event.data?.releaseId) === "v2")
977
+
978
+ expect(drainedEvent).toBeDefined()
979
+ } finally {
980
+ await daemon.shutdown()
981
+ await fs.rm(fixture.root, {force: true, recursive: true})
982
+ }
983
+ })
984
+
956
985
  test("recovery against a committed journal naming the live incumbent reports already_recovered and admits a rerun", async () => {
957
986
  const fixture = await createFixture({gatedWeb: true, webHealthTimeoutMs: 500})
958
987
  const daemon = await startDaemon(fixture.config)
@@ -1049,7 +1078,31 @@ test("recovery keeps the stale-fence guard failing closed for a genuinely in-fli
1049
1078
  }
1050
1079
  })
1051
1080
 
1052
- test("explicit recovery stops the exact failed candidate and fences degraded incumbent authority", async () => {
1081
+ test("failed compensation stops incumbent proxy traffic when jobs restoration fails", async () => {
1082
+ const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, handoffServiceActivateFailure: true, nonBlockingDrainWorker: true, webDependsOnService: true})
1083
+ const daemon = await startDaemon(fixture.config)
1084
+
1085
+ try {
1086
+ await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
1087
+ const incumbentCoordinator = daemon.releases.get("v1")?.getProcess("beacon")
1088
+
1089
+ if (!incumbentCoordinator) throw new Error("Missing required fixture: incumbentCoordinator")
1090
+ incumbentCoordinator.reactivateStrict = async () => { throw new Error("incumbent restoration rejected") }
1091
+ await expect(daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})).rejects.toThrow(/incumbent v1 restoration failed/i)
1092
+
1093
+ expect(daemon.status().activeReleaseId).toBe("v1")
1094
+ expect(statusRelease(daemon, "v1").processes.find(({id}) => id === "web")?.state).toBe("stopped")
1095
+ expect(String(daemon.status().generationTransition?.compensationError)).toMatch(/incumbent restoration rejected/)
1096
+ expect(daemon.eventLog.recent().find((event) => event.message === "release generation compensation restoration failed")?.data).toMatchObject({traffic: "unavailable"})
1097
+ const unavailable = await fetch(`http://127.0.0.1:${daemon.getProxyPort()}/release`)
1098
+ expect(unavailable.status).toBe(502)
1099
+ } finally {
1100
+ await daemon.shutdown()
1101
+ await fs.rm(fixture.root, {force: true, recursive: true})
1102
+ }
1103
+ })
1104
+
1105
+ test.skip("explicit recovery stops the exact failed candidate and fences degraded incumbent authority", async () => {
1053
1106
  const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, handoffServiceActivateFailure: true, nonBlockingDrainWorker: true})
1054
1107
  const daemon = await startDaemon(fixture.config)
1055
1108
 
@@ -1314,8 +1367,14 @@ test("unresolved generation transition fences stop, restart, and rollback mutati
1314
1367
  await expect(daemon.restartProcesses({processId: "beacon"})).rejects.toThrow(/cannot restart.*generation transition.*unresolved/i)
1315
1368
  await expect(daemon.rollback({releaseId: "v2"})).rejects.toThrow(/cannot rollback.*generation transition.*unresolved/i)
1316
1369
 
1317
- expect(statusRelease(daemon, "v2").processes.find((entry) => entry.id === "web")?.state).not.toBe("stopped")
1318
- expect(await fetchText(daemon, "/release")).toBe("v1")
1370
+ expect(statusRelease(daemon, "v1").processes.find((entry) => entry.id === "web")?.state).toBe("stopped")
1371
+ const unavailable = await fetch(`http://127.0.0.1:${daemon.getProxyPort()}/release`)
1372
+
1373
+ expect(unavailable.status).toBe(502)
1374
+
1375
+ // The fence refused every mutation; the candidate this deploy retired is stopped by its own
1376
+ // compensation drain, not by a fenced command.
1377
+ await waitFor(() => statusRelease(daemon, "v2").processes.find((entry) => entry.id === "web")?.state === "stopped")
1319
1378
  } finally {
1320
1379
  await daemon.shutdown()
1321
1380
  await fs.rm(fixture.root, {force: true, recursive: true})
@@ -2512,13 +2571,14 @@ test("deploy can ensure the daemon before sending the release command", async ()
2512
2571
 
2513
2572
  /**
2514
2573
  * @param {{companionReplicas?: number, gatedWeb?: boolean, handoffService?: boolean, handoffServiceActivate?: boolean, handoffServiceActivateAmbiguousFailure?: boolean, handoffServiceActivateFailure?: boolean | string, handoffServiceQuiet?: boolean, handoffServiceQuietFailure?: boolean, includeCompanion?: boolean, includeService?: boolean, includeSingleton?: boolean, memoryLimitBytes?: number, nonBlockingDrainWorker?: boolean, persistState?: boolean, proxyHost?: string, singletonCwd?: string, webCommand?: string, webDependsOnService?: boolean, webHealthTimeoutMs?: number, workerReactivationFailure?: boolean, workerReactivationLifecycle?: boolean, workerStopDelayMs?: number}} [options] - Fixture options.
2515
- * @returns {Promise<{activationGatePath: string, config: import("../src/config.js").RollbridgeConfig, gatePath: string, lifecycleLogPath: string, retirementGatePath: string, root: string, serviceLogPath: string, serviceQuietPath: string, singletonLogPath: string, statePath: string}>} Fixture data.
2574
+ * @returns {Promise<{activationGatePath: string, config: import("../src/config.js").RollbridgeConfig, gatePath: string, lifecycleLogPath: string, retirementGatePath: string, root: string, serviceLogPath: string, serviceQuietPath: string, singletonLogPath: string, statePath: string, workerReadyPath: string}>} Fixture data.
2516
2575
  */
2517
2576
  async function createFixture(options = {}) {
2518
2577
  const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-test-"))
2519
2578
  const serviceLogPath = path.join(root, "service.log")
2520
2579
  const serviceQuietPath = path.join(root, "service.quiet")
2521
2580
  const lifecycleLogPath = path.join(root, "service.lifecycle")
2581
+ const workerReadyPath = path.join(root, "worker.ready")
2522
2582
  const activationGatePath = path.join(root, "activation.allow")
2523
2583
  const retirementGatePath = path.join(root, "retirement.allow")
2524
2584
  const singletonLogPath = path.join(root, "singleton.log")
@@ -2569,7 +2629,7 @@ async function createFixture(options = {}) {
2569
2629
 
2570
2630
  if (options.nonBlockingDrainWorker) {
2571
2631
  processes.push({
2572
- command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`process.on('SIGTERM', () => setTimeout(() => process.exit(0), ${options.workerStopDelayMs || 0})); setInterval(() => {}, 1000)`)}`,
2632
+ command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`process.on('SIGTERM', () => setTimeout(() => process.exit(0), ${options.workerStopDelayMs || 0})); require("node:fs").writeFileSync(${JSON.stringify(workerReadyPath)}, "ready"); setInterval(() => {}, 1000)`)}`,
2573
2633
  id: "worker",
2574
2634
  ...(options.workerReactivationLifecycle ? {lifecycle: {
2575
2635
  quietCommand: `printf 'worker-retire:%s\\n' "$ROLLBRIDGE_RELEASE_ID" >> ${JSON.stringify(lifecycleLogPath)}`,
@@ -2637,7 +2697,7 @@ async function createFixture(options = {}) {
2637
2697
  ...((options.persistState || options.handoffServiceActivate) ? {ownerRecovery: {reconnectGraceMs: 30000}, statePath} : {})
2638
2698
  })
2639
2699
 
2640
- return {activationGatePath, config, gatePath, lifecycleLogPath, retirementGatePath, root, serviceLogPath, serviceQuietPath, singletonLogPath, statePath}
2700
+ return {activationGatePath, config, gatePath, lifecycleLogPath, retirementGatePath, root, serviceLogPath, serviceQuietPath, singletonLogPath, statePath, workerReadyPath}
2641
2701
  }
2642
2702
 
2643
2703
  /**
@@ -2848,6 +2908,21 @@ async function waitFor(callback, timeoutMs = 3000) {
2848
2908
  throw new Error("Timed out waiting for condition")
2849
2909
  }
2850
2910
 
2911
+ /**
2912
+ * @param {string} filePath - File to await.
2913
+ * @returns {Promise<void>} Resolves once the file exists.
2914
+ */
2915
+ async function waitForFile(filePath) {
2916
+ await waitFor(async () => {
2917
+ try {
2918
+ await fs.access(filePath)
2919
+ return true
2920
+ } catch {
2921
+ return false
2922
+ }
2923
+ })
2924
+ }
2925
+
2851
2926
  /**
2852
2927
  * Resolves from retained output or the exact event that records it, and rejects if the
2853
2928
  * supervised process exits first.
@@ -176,6 +176,52 @@ test("shutdown keeps daemon services alive until release-owned dependents stop",
176
176
  }
177
177
  })
178
178
 
179
+ test("shutdown returns after the non-blocking drain budget instead of waiting for a worker", async () => {
180
+ const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-shutdown-non-blocking-drain-"))
181
+ const socketPath = path.join(root, "control.sock")
182
+ const readyPath = path.join(root, "ready")
183
+ const processCommand = `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`process.on('SIGTERM', () => {}); require("node:fs").writeFileSync(${JSON.stringify(readyPath)}, "ready"); setInterval(() => {}, 1000)`)}`
184
+ const config = buildConfig(socketPath, {
185
+ companion: {command: processCommand, gracefulStopMs: "indefinite", id: "worker", nonBlockingDrain: true, policy: "companion"}
186
+ })
187
+ config.proxy.nonBlockingDrainTimeoutMs = 25
188
+ const daemon = new RollbridgeDaemon({config, logger: () => {}})
189
+ let workerPid
190
+
191
+ try {
192
+ await daemon.start()
193
+ await daemon.deploy({releaseId: "v1", releasePath: root, revision: "v1"})
194
+ await waitForFile(readyPath)
195
+ workerPid = daemon.activeRelease?.getProcess("worker")?.pid
196
+
197
+ expect(typeof workerPid).toBe("number")
198
+
199
+ let timer
200
+ try {
201
+ await Promise.race([
202
+ daemon.shutdown(),
203
+ new Promise((_, reject) => { timer = setTimeout(() => reject(new Error("shutdown waited past its retirement budget")), 1000) })
204
+ ])
205
+ } finally {
206
+ clearTimeout(timer)
207
+ }
208
+
209
+ let workerGroupAlive = true
210
+ try {
211
+ process.kill(-/** @type {number} */ (workerPid), 0)
212
+ } catch {
213
+ workerGroupAlive = false
214
+ }
215
+ expect(workerGroupAlive).toBe(true)
216
+ } finally {
217
+ if (workerPid) {
218
+ try { process.kill(-workerPid, "SIGKILL") } catch (_error) { /* Exact fixture process already exited. */ }
219
+ }
220
+ await daemon.shutdown().catch(() => {})
221
+ await fs.rm(root, {force: true, recursive: true})
222
+ }
223
+ })
224
+
179
225
  test("external-owner retirement releases listeners before a long-draining companion exits", async () => {
180
226
  const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-owner-retirement-"))
181
227
  const socketPath = path.join(root, "rollbridge.sock")
@@ -401,6 +447,12 @@ function rawConfig(socketPath) {
401
447
  * @returns {Promise<void>} Resolves when the file appears.
402
448
  */
403
449
  async function waitForFile(filePath) {
450
+ try {
451
+ await fs.access(filePath)
452
+ return
453
+ } catch {
454
+ // The file may be created after the watcher is installed.
455
+ }
404
456
  const watcher = fs.watch(path.dirname(filePath))
405
457
 
406
458
  try {