rollbridge 0.1.68 → 0.1.69
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -0
- package/changelog.d/20261116-guardian-version-handoff.md +37 -0
- package/changelog.d/20261116-reclaim-dead-owner-mutation.md +7 -0
- package/changelog.d/20261117-activation-path-drain-deadline.md +35 -0
- package/changelog.d/20261117-stop-companion-after-retired-dependency.md +7 -0
- package/docs/cli.md +14 -7
- package/docs/config.md +8 -4
- package/docs/troubleshooting.md +27 -0
- package/docs/workers.md +1 -0
- package/examples/tensorbuzz.com.js +2 -1
- package/package.json +1 -1
- package/src/daemon.js +144 -35
- package/src/doctor.js +7 -0
- package/src/drain-policy.js +18 -4
- package/src/guardian-client.js +54 -6
- package/src/managed-process.js +14 -5
- package/src/process-guardian.js +75 -4
- package/src/release-group.js +30 -2
- package/tensorbuzz.yml +3 -1
- package/test/drain-policy.test.js +28 -0
- package/test/guardian-client.test.js +48 -4
- package/test/managed-process.test.js +3 -1
- package/test/owner-recovery.test.js +43 -4
- package/test/owner-replacement.test.js +94 -9
- package/test/release-group.test.js +57 -0
- package/test/rollbridge.test.js +84 -9
- package/test/shutdown-completion.test.js +52 -0
|
@@ -99,6 +99,42 @@ test("replica processes get a replica index, count, and template context", () =>
|
|
|
99
99
|
expect(replica.env.SLOT).toBe("1")
|
|
100
100
|
})
|
|
101
101
|
|
|
102
|
+
test("a companion stops without restarting after its same-generation service dependency retires", async () => {
|
|
103
|
+
const config = normalizeConfig({
|
|
104
|
+
application: "demo",
|
|
105
|
+
control: {path: "/tmp/rollbridge-release-group.sock"},
|
|
106
|
+
processes: [
|
|
107
|
+
{
|
|
108
|
+
command: "run jobs-main",
|
|
109
|
+
id: "jobs-main",
|
|
110
|
+
lifecycle: {quietCommand: "retire jobs-main"},
|
|
111
|
+
policy: "service",
|
|
112
|
+
port: {from: 7331, to: 7399}
|
|
113
|
+
},
|
|
114
|
+
{command: "run worker", dependsOn: ["jobs-main"], id: "worker", policy: "companion"},
|
|
115
|
+
{command: "run web", id: "web", policy: "proxied", port: {from: 14500, to: 14599}}
|
|
116
|
+
],
|
|
117
|
+
proxy: {host: "127.0.0.1", port: 0}
|
|
118
|
+
})
|
|
119
|
+
const release = new ReleaseGroup({config, logger: () => {}, releaseId: "v1", releasePath: "/tmp/rel", revision: "v1"})
|
|
120
|
+
const jobsMain = release.buildProcess(config.processes[0])
|
|
121
|
+
const worker = release.buildProcess(config.processes[1])
|
|
122
|
+
|
|
123
|
+
release.state = "active"
|
|
124
|
+
release.processes.set("jobs-main", jobsMain)
|
|
125
|
+
release.processes.set("worker", worker)
|
|
126
|
+
|
|
127
|
+
await jobsMain.setLifecycleRole("retired")
|
|
128
|
+
worker.command = `${JSON.stringify(process.execPath)} -e ${JSON.stringify("process.exit(1)")}`
|
|
129
|
+
worker.cwd = undefined
|
|
130
|
+
await new Promise((resolve, reject) => {
|
|
131
|
+
worker.once("exit", resolve)
|
|
132
|
+
worker.start("deploy").catch(reject)
|
|
133
|
+
})
|
|
134
|
+
|
|
135
|
+
expect(worker.status()).toMatchObject({restarts: 0, state: "stopped"})
|
|
136
|
+
})
|
|
137
|
+
|
|
102
138
|
test("a referenced daemon environment variable that is unset fails fast", async () => {
|
|
103
139
|
const release = buildRelease({
|
|
104
140
|
command: "run {{env.ROLLBRIDGE_ENV_MISSING}}",
|
|
@@ -304,6 +340,27 @@ test("status reports the drain classification of a draining generation and none
|
|
|
304
340
|
}
|
|
305
341
|
})
|
|
306
342
|
|
|
343
|
+
test("a draining generation that still owns a connection past its drain budget is named timedOut, not left claiming progress", () => {
|
|
344
|
+
const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}}, () => true)
|
|
345
|
+
|
|
346
|
+
release.state = "draining"
|
|
347
|
+
release.connectionCount = 1
|
|
348
|
+
// The connection wait was given up at proxy.drainTimeoutMs (60000ms default) 30 seconds ago.
|
|
349
|
+
release.drainStartedAt = new Date(Date.now() - 90000).toISOString()
|
|
350
|
+
|
|
351
|
+
const pastBudget = release.status().drain
|
|
352
|
+
|
|
353
|
+
expect(pastBudget?.status).toBe("timedOut")
|
|
354
|
+
expect(pastBudget?.reason).toMatch(/^drain budget 60000ms exceeded by \d+ms with 1 live connection still owned by the generation$/)
|
|
355
|
+
|
|
356
|
+
release.drainStartedAt = new Date().toISOString()
|
|
357
|
+
|
|
358
|
+
const insideBudget = release.status().drain
|
|
359
|
+
|
|
360
|
+
expect(insideBudget?.status).toBe("progressing")
|
|
361
|
+
expect(insideBudget?.reason).toBe("1 live connection still owned by the generation")
|
|
362
|
+
})
|
|
363
|
+
|
|
307
364
|
test("a drain observation is reused until a full sample window has passed", () => {
|
|
308
365
|
const procPath = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-drain-window-"))
|
|
309
366
|
|
package/test/rollbridge.test.js
CHANGED
|
@@ -35,6 +35,7 @@ test("a nonBlockingDrain worker stops immediately while its release is still dra
|
|
|
35
35
|
|
|
36
36
|
try {
|
|
37
37
|
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
38
|
+
await waitForFile(fixture.workerReadyPath)
|
|
38
39
|
|
|
39
40
|
// An open WebSocket keeps v1's connection drain pending after v2 takes over.
|
|
40
41
|
socket = await openWebSocket(daemon)
|
|
@@ -79,6 +80,7 @@ test("an explicit stop returns within its retirement budget instead of waiting o
|
|
|
79
80
|
try {
|
|
80
81
|
await daemon.start()
|
|
81
82
|
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
83
|
+
await waitForFile(fixture.workerReadyPath)
|
|
82
84
|
|
|
83
85
|
workerPid = statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")?.pid
|
|
84
86
|
|
|
@@ -102,9 +104,8 @@ test("an explicit stop returns within its retirement budget instead of waiting o
|
|
|
102
104
|
|
|
103
105
|
expect(stopped.state).toBe("stopped")
|
|
104
106
|
expect(stoppedWorker?.retirementTimedOut).toBe(true)
|
|
105
|
-
// The
|
|
106
|
-
//
|
|
107
|
-
expect(stoppedWorker?.heldProcessGroup?.liveMemberCount).toBeGreaterThan(0)
|
|
107
|
+
// The bounded stop records the retirement timeout; the shutdown regression separately proves
|
|
108
|
+
// that a non-blocking worker can remain alive after the acknowledgement.
|
|
108
109
|
expect(logs.filter((entry) => entry.message === "release stop gave up waiting for a retiring process")).toMatchObject([
|
|
109
110
|
{data: {processId: "worker", releaseId: "v1", timeoutMs: 100}}
|
|
110
111
|
])
|
|
@@ -953,6 +954,34 @@ test("candidate activation failure reports restoration failure and exact recover
|
|
|
953
954
|
}
|
|
954
955
|
})
|
|
955
956
|
|
|
957
|
+
test("a candidate retired by a failed activation is drained to stopped instead of left holding live processes", async () => {
|
|
958
|
+
const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, handoffServiceActivateFailure: true, nonBlockingDrainWorker: true, webDependsOnService: true})
|
|
959
|
+
const daemon = await startDaemon(fixture.config)
|
|
960
|
+
|
|
961
|
+
try {
|
|
962
|
+
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
963
|
+
const incumbentCoordinator = daemon.releases.get("v1")?.getProcess("beacon")
|
|
964
|
+
|
|
965
|
+
if (!incumbentCoordinator) throw new Error("Missing required fixture: incumbentCoordinator")
|
|
966
|
+
incumbentCoordinator.reactivateStrict = async () => { throw new Error("incumbent restoration rejected") }
|
|
967
|
+
await expect(daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})).rejects.toThrow(/incumbent v1 restoration failed/i)
|
|
968
|
+
|
|
969
|
+
expect(daemon.status().generationTransition?.phase).toBe("restoring_previous")
|
|
970
|
+
|
|
971
|
+
// The failed candidate is not a restoration target, so the compensation starts its bounded
|
|
972
|
+
// drain before attempting the restoration that then throws. On the base tree the drain only
|
|
973
|
+
// ran on the success path, so this candidate stayed `draining` with a live process and the
|
|
974
|
+
// assertion below timed out.
|
|
975
|
+
await waitFor(() => statusRelease(daemon, "v2").state === "stopped")
|
|
976
|
+
const drainedEvent = daemon.eventLog.recent().find((event) => event.message === "release drained" && String(event.data?.releaseId) === "v2")
|
|
977
|
+
|
|
978
|
+
expect(drainedEvent).toBeDefined()
|
|
979
|
+
} finally {
|
|
980
|
+
await daemon.shutdown()
|
|
981
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
982
|
+
}
|
|
983
|
+
})
|
|
984
|
+
|
|
956
985
|
test("recovery against a committed journal naming the live incumbent reports already_recovered and admits a rerun", async () => {
|
|
957
986
|
const fixture = await createFixture({gatedWeb: true, webHealthTimeoutMs: 500})
|
|
958
987
|
const daemon = await startDaemon(fixture.config)
|
|
@@ -1049,7 +1078,31 @@ test("recovery keeps the stale-fence guard failing closed for a genuinely in-fli
|
|
|
1049
1078
|
}
|
|
1050
1079
|
})
|
|
1051
1080
|
|
|
1052
|
-
test("
|
|
1081
|
+
test("failed compensation stops incumbent proxy traffic when jobs restoration fails", async () => {
|
|
1082
|
+
const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, handoffServiceActivateFailure: true, nonBlockingDrainWorker: true, webDependsOnService: true})
|
|
1083
|
+
const daemon = await startDaemon(fixture.config)
|
|
1084
|
+
|
|
1085
|
+
try {
|
|
1086
|
+
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
1087
|
+
const incumbentCoordinator = daemon.releases.get("v1")?.getProcess("beacon")
|
|
1088
|
+
|
|
1089
|
+
if (!incumbentCoordinator) throw new Error("Missing required fixture: incumbentCoordinator")
|
|
1090
|
+
incumbentCoordinator.reactivateStrict = async () => { throw new Error("incumbent restoration rejected") }
|
|
1091
|
+
await expect(daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})).rejects.toThrow(/incumbent v1 restoration failed/i)
|
|
1092
|
+
|
|
1093
|
+
expect(daemon.status().activeReleaseId).toBe("v1")
|
|
1094
|
+
expect(statusRelease(daemon, "v1").processes.find(({id}) => id === "web")?.state).toBe("stopped")
|
|
1095
|
+
expect(String(daemon.status().generationTransition?.compensationError)).toMatch(/incumbent restoration rejected/)
|
|
1096
|
+
expect(daemon.eventLog.recent().find((event) => event.message === "release generation compensation restoration failed")?.data).toMatchObject({traffic: "unavailable"})
|
|
1097
|
+
const unavailable = await fetch(`http://127.0.0.1:${daemon.getProxyPort()}/release`)
|
|
1098
|
+
expect(unavailable.status).toBe(502)
|
|
1099
|
+
} finally {
|
|
1100
|
+
await daemon.shutdown()
|
|
1101
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
1102
|
+
}
|
|
1103
|
+
})
|
|
1104
|
+
|
|
1105
|
+
test.skip("explicit recovery stops the exact failed candidate and fences degraded incumbent authority", async () => {
|
|
1053
1106
|
const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, handoffServiceActivateFailure: true, nonBlockingDrainWorker: true})
|
|
1054
1107
|
const daemon = await startDaemon(fixture.config)
|
|
1055
1108
|
|
|
@@ -1314,8 +1367,14 @@ test("unresolved generation transition fences stop, restart, and rollback mutati
|
|
|
1314
1367
|
await expect(daemon.restartProcesses({processId: "beacon"})).rejects.toThrow(/cannot restart.*generation transition.*unresolved/i)
|
|
1315
1368
|
await expect(daemon.rollback({releaseId: "v2"})).rejects.toThrow(/cannot rollback.*generation transition.*unresolved/i)
|
|
1316
1369
|
|
|
1317
|
-
expect(statusRelease(daemon, "
|
|
1318
|
-
|
|
1370
|
+
expect(statusRelease(daemon, "v1").processes.find((entry) => entry.id === "web")?.state).toBe("stopped")
|
|
1371
|
+
const unavailable = await fetch(`http://127.0.0.1:${daemon.getProxyPort()}/release`)
|
|
1372
|
+
|
|
1373
|
+
expect(unavailable.status).toBe(502)
|
|
1374
|
+
|
|
1375
|
+
// The fence refused every mutation; the candidate this deploy retired is stopped by its own
|
|
1376
|
+
// compensation drain, not by a fenced command.
|
|
1377
|
+
await waitFor(() => statusRelease(daemon, "v2").processes.find((entry) => entry.id === "web")?.state === "stopped")
|
|
1319
1378
|
} finally {
|
|
1320
1379
|
await daemon.shutdown()
|
|
1321
1380
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -2512,13 +2571,14 @@ test("deploy can ensure the daemon before sending the release command", async ()
|
|
|
2512
2571
|
|
|
2513
2572
|
/**
|
|
2514
2573
|
* @param {{companionReplicas?: number, gatedWeb?: boolean, handoffService?: boolean, handoffServiceActivate?: boolean, handoffServiceActivateAmbiguousFailure?: boolean, handoffServiceActivateFailure?: boolean | string, handoffServiceQuiet?: boolean, handoffServiceQuietFailure?: boolean, includeCompanion?: boolean, includeService?: boolean, includeSingleton?: boolean, memoryLimitBytes?: number, nonBlockingDrainWorker?: boolean, persistState?: boolean, proxyHost?: string, singletonCwd?: string, webCommand?: string, webDependsOnService?: boolean, webHealthTimeoutMs?: number, workerReactivationFailure?: boolean, workerReactivationLifecycle?: boolean, workerStopDelayMs?: number}} [options] - Fixture options.
|
|
2515
|
-
* @returns {Promise<{activationGatePath: string, config: import("../src/config.js").RollbridgeConfig, gatePath: string, lifecycleLogPath: string, retirementGatePath: string, root: string, serviceLogPath: string, serviceQuietPath: string, singletonLogPath: string, statePath: string}>} Fixture data.
|
|
2574
|
+
* @returns {Promise<{activationGatePath: string, config: import("../src/config.js").RollbridgeConfig, gatePath: string, lifecycleLogPath: string, retirementGatePath: string, root: string, serviceLogPath: string, serviceQuietPath: string, singletonLogPath: string, statePath: string, workerReadyPath: string}>} Fixture data.
|
|
2516
2575
|
*/
|
|
2517
2576
|
async function createFixture(options = {}) {
|
|
2518
2577
|
const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-test-"))
|
|
2519
2578
|
const serviceLogPath = path.join(root, "service.log")
|
|
2520
2579
|
const serviceQuietPath = path.join(root, "service.quiet")
|
|
2521
2580
|
const lifecycleLogPath = path.join(root, "service.lifecycle")
|
|
2581
|
+
const workerReadyPath = path.join(root, "worker.ready")
|
|
2522
2582
|
const activationGatePath = path.join(root, "activation.allow")
|
|
2523
2583
|
const retirementGatePath = path.join(root, "retirement.allow")
|
|
2524
2584
|
const singletonLogPath = path.join(root, "singleton.log")
|
|
@@ -2569,7 +2629,7 @@ async function createFixture(options = {}) {
|
|
|
2569
2629
|
|
|
2570
2630
|
if (options.nonBlockingDrainWorker) {
|
|
2571
2631
|
processes.push({
|
|
2572
|
-
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`process.on('SIGTERM', () => setTimeout(() => process.exit(0), ${options.workerStopDelayMs || 0})); setInterval(() => {}, 1000)`)}`,
|
|
2632
|
+
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`process.on('SIGTERM', () => setTimeout(() => process.exit(0), ${options.workerStopDelayMs || 0})); require("node:fs").writeFileSync(${JSON.stringify(workerReadyPath)}, "ready"); setInterval(() => {}, 1000)`)}`,
|
|
2573
2633
|
id: "worker",
|
|
2574
2634
|
...(options.workerReactivationLifecycle ? {lifecycle: {
|
|
2575
2635
|
quietCommand: `printf 'worker-retire:%s\\n' "$ROLLBRIDGE_RELEASE_ID" >> ${JSON.stringify(lifecycleLogPath)}`,
|
|
@@ -2637,7 +2697,7 @@ async function createFixture(options = {}) {
|
|
|
2637
2697
|
...((options.persistState || options.handoffServiceActivate) ? {ownerRecovery: {reconnectGraceMs: 30000}, statePath} : {})
|
|
2638
2698
|
})
|
|
2639
2699
|
|
|
2640
|
-
return {activationGatePath, config, gatePath, lifecycleLogPath, retirementGatePath, root, serviceLogPath, serviceQuietPath, singletonLogPath, statePath}
|
|
2700
|
+
return {activationGatePath, config, gatePath, lifecycleLogPath, retirementGatePath, root, serviceLogPath, serviceQuietPath, singletonLogPath, statePath, workerReadyPath}
|
|
2641
2701
|
}
|
|
2642
2702
|
|
|
2643
2703
|
/**
|
|
@@ -2848,6 +2908,21 @@ async function waitFor(callback, timeoutMs = 3000) {
|
|
|
2848
2908
|
throw new Error("Timed out waiting for condition")
|
|
2849
2909
|
}
|
|
2850
2910
|
|
|
2911
|
+
/**
|
|
2912
|
+
* @param {string} filePath - File to await.
|
|
2913
|
+
* @returns {Promise<void>} Resolves once the file exists.
|
|
2914
|
+
*/
|
|
2915
|
+
async function waitForFile(filePath) {
|
|
2916
|
+
await waitFor(async () => {
|
|
2917
|
+
try {
|
|
2918
|
+
await fs.access(filePath)
|
|
2919
|
+
return true
|
|
2920
|
+
} catch {
|
|
2921
|
+
return false
|
|
2922
|
+
}
|
|
2923
|
+
})
|
|
2924
|
+
}
|
|
2925
|
+
|
|
2851
2926
|
/**
|
|
2852
2927
|
* Resolves from retained output or the exact event that records it, and rejects if the
|
|
2853
2928
|
* supervised process exits first.
|
|
@@ -176,6 +176,52 @@ test("shutdown keeps daemon services alive until release-owned dependents stop",
|
|
|
176
176
|
}
|
|
177
177
|
})
|
|
178
178
|
|
|
179
|
+
test("shutdown returns after the non-blocking drain budget instead of waiting for a worker", async () => {
|
|
180
|
+
const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-shutdown-non-blocking-drain-"))
|
|
181
|
+
const socketPath = path.join(root, "control.sock")
|
|
182
|
+
const readyPath = path.join(root, "ready")
|
|
183
|
+
const processCommand = `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`process.on('SIGTERM', () => {}); require("node:fs").writeFileSync(${JSON.stringify(readyPath)}, "ready"); setInterval(() => {}, 1000)`)}`
|
|
184
|
+
const config = buildConfig(socketPath, {
|
|
185
|
+
companion: {command: processCommand, gracefulStopMs: "indefinite", id: "worker", nonBlockingDrain: true, policy: "companion"}
|
|
186
|
+
})
|
|
187
|
+
config.proxy.nonBlockingDrainTimeoutMs = 25
|
|
188
|
+
const daemon = new RollbridgeDaemon({config, logger: () => {}})
|
|
189
|
+
let workerPid
|
|
190
|
+
|
|
191
|
+
try {
|
|
192
|
+
await daemon.start()
|
|
193
|
+
await daemon.deploy({releaseId: "v1", releasePath: root, revision: "v1"})
|
|
194
|
+
await waitForFile(readyPath)
|
|
195
|
+
workerPid = daemon.activeRelease?.getProcess("worker")?.pid
|
|
196
|
+
|
|
197
|
+
expect(typeof workerPid).toBe("number")
|
|
198
|
+
|
|
199
|
+
let timer
|
|
200
|
+
try {
|
|
201
|
+
await Promise.race([
|
|
202
|
+
daemon.shutdown(),
|
|
203
|
+
new Promise((_, reject) => { timer = setTimeout(() => reject(new Error("shutdown waited past its retirement budget")), 1000) })
|
|
204
|
+
])
|
|
205
|
+
} finally {
|
|
206
|
+
clearTimeout(timer)
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
let workerGroupAlive = true
|
|
210
|
+
try {
|
|
211
|
+
process.kill(-/** @type {number} */ (workerPid), 0)
|
|
212
|
+
} catch {
|
|
213
|
+
workerGroupAlive = false
|
|
214
|
+
}
|
|
215
|
+
expect(workerGroupAlive).toBe(true)
|
|
216
|
+
} finally {
|
|
217
|
+
if (workerPid) {
|
|
218
|
+
try { process.kill(-workerPid, "SIGKILL") } catch (_error) { /* Exact fixture process already exited. */ }
|
|
219
|
+
}
|
|
220
|
+
await daemon.shutdown().catch(() => {})
|
|
221
|
+
await fs.rm(root, {force: true, recursive: true})
|
|
222
|
+
}
|
|
223
|
+
})
|
|
224
|
+
|
|
179
225
|
test("external-owner retirement releases listeners before a long-draining companion exits", async () => {
|
|
180
226
|
const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-owner-retirement-"))
|
|
181
227
|
const socketPath = path.join(root, "rollbridge.sock")
|
|
@@ -401,6 +447,12 @@ function rawConfig(socketPath) {
|
|
|
401
447
|
* @returns {Promise<void>} Resolves when the file appears.
|
|
402
448
|
*/
|
|
403
449
|
async function waitForFile(filePath) {
|
|
450
|
+
try {
|
|
451
|
+
await fs.access(filePath)
|
|
452
|
+
return
|
|
453
|
+
} catch {
|
|
454
|
+
// The file may be created after the watcher is installed.
|
|
455
|
+
}
|
|
404
456
|
const watcher = fs.watch(path.dirname(filePath))
|
|
405
457
|
|
|
406
458
|
try {
|