rollbridge 0.1.70 → 0.1.71
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -63
- package/TODO.md +0 -13
- package/docs/cli.md +3 -128
- package/docs/config.md +6 -177
- package/docs/troubleshooting.md +6 -97
- package/docs/velocious.md +1 -1
- package/docs/workers.md +0 -11
- package/examples/tensorbuzz.com.js +2 -3
- package/package.json +1 -1
- package/src/cli.js +2 -68
- package/src/config.js +8 -347
- package/src/daemon.js +65 -387
- package/src/doctor.js +0 -74
- package/src/guardian-client.js +8 -67
- package/src/health.js +8 -99
- package/src/managed-process.js +16 -197
- package/src/process-guardian.js +22 -212
- package/src/process-memory.js +2 -62
- package/src/release-group.js +37 -435
- package/tensorbuzz.yml +1 -20
- package/test/completion.test.js +2 -5
- package/test/config-validation.test.js +5 -135
- package/test/guardian-client.test.js +5 -119
- package/test/health.test.js +1 -74
- package/test/managed-process.test.js +1 -260
- package/test/owner-recovery.test.js +21 -125
- package/test/owner-replacement.test.js +14 -175
- package/test/process-memory.test.js +1 -47
- package/test/release-group.test.js +2 -408
- package/test/rollbridge.test.js +72 -737
- package/test/shutdown-completion.test.js +0 -52
- package/test/support/process.js +1 -3
- package/changelog.d/20261025-guardian-re-register-dead-generation.md +0 -13
- package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +0 -1
- package/changelog.d/20261104-compose-service-units.md +0 -21
- package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +0 -20
- package/changelog.d/20261106-release-dependency-boot-budgets.md +0 -32
- package/changelog.d/20261109-path-filtered-builds.md +0 -11
- package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +0 -26
- package/changelog.d/20261111-contention-sensitive-daemon-specs.md +0 -27
- package/changelog.d/20261113-bounded-nonblocking-retirement.md +0 -5
- package/changelog.d/20261115-bound-draining-release-groups.md +0 -62
- package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +0 -27
- package/changelog.d/20261115-wedged-generation-escalation.md +0 -44
- package/changelog.d/20261116-bounded-explicit-stop-retirement.md +0 -30
- package/changelog.d/20261116-config-json-canonical-shape.md +0 -11
- package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +0 -29
- package/changelog.d/20261116-guardian-version-handoff.md +0 -37
- package/changelog.d/20261116-reclaim-dead-owner-mutation.md +0 -7
- package/changelog.d/20261117-activate-before-incumbent-retirement.md +0 -1
- package/changelog.d/20261117-activation-path-drain-deadline.md +0 -35
- package/changelog.d/20261117-stop-companion-after-retired-dependency.md +0 -7
- package/docs/compose-services.md +0 -222
- package/src/compose-unit.js +0 -725
- package/src/drain-policy.js +0 -252
- package/src/process-order.js +0 -26
- package/test/compose-config.test.js +0 -160
- package/test/compose-docker.test.js +0 -297
- package/test/compose-unit.test.js +0 -415
- package/test/drain-policy.test.js +0 -328
- package/test/fixtures/deferred-boot-app.js +0 -106
- package/test/fixtures/gated-app.js +0 -50
- package/test/tensorbuzz-config.test.js +0 -228
package/test/rollbridge.test.js
CHANGED
|
@@ -19,12 +19,10 @@ describe("rollbridge", () => {
|
|
|
19
19
|
const currentDir = path.dirname(fileURLToPath(import.meta.url))
|
|
20
20
|
const binPath = path.join(currentDir, "..", "bin", "rollbridge")
|
|
21
21
|
const dependentAppPath = path.join(currentDir, "fixtures", "dependent-app.js")
|
|
22
|
-
const deferredBootAppPath = path.join(currentDir, "fixtures", "deferred-boot-app.js")
|
|
23
22
|
const dummyAppPath = path.join(currentDir, "fixtures", "dummy-app.js")
|
|
24
23
|
const memoryHogPath = path.join(currentDir, "fixtures", "memory-hog.js")
|
|
25
24
|
const serviceAppPath = path.join(currentDir, "fixtures", "service-app.js")
|
|
26
25
|
const singletonAppPath = path.join(currentDir, "fixtures", "singleton-app.js")
|
|
27
|
-
const gatedAppPath = path.join(currentDir, "fixtures", "gated-app.js")
|
|
28
26
|
const linuxTest = process.platform === "linux" ? test : test.skip
|
|
29
27
|
|
|
30
28
|
test("a nonBlockingDrain worker stops immediately while its release is still draining", async () => {
|
|
@@ -61,63 +59,6 @@ test("a nonBlockingDrain worker stops immediately while its release is still dra
|
|
|
61
59
|
}
|
|
62
60
|
})
|
|
63
61
|
|
|
64
|
-
test("an explicit stop returns within its retirement budget instead of waiting on a worker that is still draining", async () => {
|
|
65
|
-
const fixture = await createFixture({nonBlockingDrainWorker: true, workerStopDelayMs: 30000})
|
|
66
|
-
const config = normalizeConfig({
|
|
67
|
-
...fixture.config,
|
|
68
|
-
processes: fixture.config.processes.map((processConfig) => processConfig.id === "worker"
|
|
69
|
-
? {...processConfig, gracefulStopMs: "indefinite"}
|
|
70
|
-
: processConfig),
|
|
71
|
-
proxy: {...fixture.config.proxy, nonBlockingDrainTimeoutMs: 100}
|
|
72
|
-
})
|
|
73
|
-
/** @type {Array<{message: string, data?: import("../src/json.js").JsonValue}>} */
|
|
74
|
-
const logs = []
|
|
75
|
-
const daemon = new RollbridgeDaemon({config, logger: (message, data) => logs.push({message, data})})
|
|
76
|
-
/** @type {number | undefined} */
|
|
77
|
-
let workerPid
|
|
78
|
-
|
|
79
|
-
try {
|
|
80
|
-
await daemon.start()
|
|
81
|
-
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
82
|
-
|
|
83
|
-
workerPid = statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")?.pid
|
|
84
|
-
|
|
85
|
-
if (typeof workerPid !== "number") throw new Error("Expected the worker to be running before the stop")
|
|
86
|
-
|
|
87
|
-
const startedAtMs = Date.now()
|
|
88
|
-
|
|
89
|
-
await daemon.stopRelease("v1")
|
|
90
|
-
|
|
91
|
-
// Before this the command had no deadline for a worker whose graceful window is "indefinite",
|
|
92
|
-
// so it never returned and held the owner-mutation fence for the whole drain, refusing every
|
|
93
|
-
// later deploy for hours (TensorBuzz production, 2026-11-16). The bound is far below the
|
|
94
|
-
// worker's own 30 000 ms stop delay because the stop must still let the generation's other
|
|
95
|
-
// processes exit; what it must not do is wait for the worker at all.
|
|
96
|
-
expect(Date.now() - startedAtMs).toBeLessThan(10000)
|
|
97
|
-
|
|
98
|
-
await retirementRecordSettled(daemon, "v1", "worker")
|
|
99
|
-
|
|
100
|
-
const stopped = statusRelease(daemon, "v1")
|
|
101
|
-
const stoppedWorker = stopped.processes.find((processStatus) => processStatus.id === "worker")
|
|
102
|
-
|
|
103
|
-
expect(stopped.state).toBe("stopped")
|
|
104
|
-
expect(stoppedWorker?.retirementTimedOut).toBe(true)
|
|
105
|
-
// The tracked record is the shell wrapper, which is already gone; the worker it launched is
|
|
106
|
-
// still draining inside the same process group, and that is the residue the stop must report.
|
|
107
|
-
expect(stoppedWorker?.heldProcessGroup?.liveMemberCount).toBeGreaterThan(0)
|
|
108
|
-
expect(logs.filter((entry) => entry.message === "release stop gave up waiting for a retiring process")).toMatchObject([
|
|
109
|
-
{data: {processId: "worker", releaseId: "v1", timeoutMs: 100}}
|
|
110
|
-
])
|
|
111
|
-
expect(logs.filter((entry) => entry.message === "release stopped")).toMatchObject([{data: {releaseId: "v1"}}])
|
|
112
|
-
} finally {
|
|
113
|
-
await daemon.shutdown()
|
|
114
|
-
if (workerPid !== undefined) {
|
|
115
|
-
try { process.kill(-workerPid, "SIGKILL") } catch (_error) { /* The group already exited. */ }
|
|
116
|
-
}
|
|
117
|
-
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
118
|
-
}
|
|
119
|
-
})
|
|
120
|
-
|
|
121
62
|
test("deploy switches new HTTP traffic while old WebSockets drain", async () => {
|
|
122
63
|
const fixture = await createFixture()
|
|
123
64
|
const daemon = await startDaemon(fixture.config)
|
|
@@ -143,6 +84,27 @@ test("deploy switches new HTTP traffic while old WebSockets drain", async () =>
|
|
|
143
84
|
}
|
|
144
85
|
})
|
|
145
86
|
|
|
87
|
+
test("deploy does not wait for retirement and force-stops an old generation by the drain deadline", async () => {
|
|
88
|
+
const fixture = await createFixture({nonBlockingDrainWorker: true, workerStopDelayMs: 10000})
|
|
89
|
+
const config = normalizeConfig({...fixture.config, proxy: {...fixture.config.proxy, drainTimeoutMs: 50, forceStopTimeoutMs: 50}})
|
|
90
|
+
const daemon = await startDaemon(config)
|
|
91
|
+
|
|
92
|
+
try {
|
|
93
|
+
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
94
|
+
const startedAt = Date.now()
|
|
95
|
+
|
|
96
|
+
await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
97
|
+
|
|
98
|
+
expect(Date.now() - startedAt).toBeLessThan(1000)
|
|
99
|
+
expect(await fetchText(daemon, "/release")).toBe("v2")
|
|
100
|
+
await waitFor(() => statusRelease(daemon, "v1").state === "stopped", 8000)
|
|
101
|
+
expect(statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")?.state).toBe("stopped")
|
|
102
|
+
} finally {
|
|
103
|
+
await daemon.shutdown()
|
|
104
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
105
|
+
}
|
|
106
|
+
})
|
|
107
|
+
|
|
146
108
|
test("failed health check leaves the previous release active", async () => {
|
|
147
109
|
const fixture = await createFixture()
|
|
148
110
|
const daemon = await startDaemon(fixture.config)
|
|
@@ -184,7 +146,7 @@ test("deploy reloads process config and retires the previous worker with the ref
|
|
|
184
146
|
|
|
185
147
|
await writeConfigFile(refreshedConfig, fixture.root)
|
|
186
148
|
await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
187
|
-
await waitFor(() => statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")?.state === "stopped",
|
|
149
|
+
await waitFor(() => statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")?.state === "stopped", 5000)
|
|
188
150
|
|
|
189
151
|
expect(daemon.config.processes.find((processConfig) => processConfig.id === "worker")?.gracefulStopMs).toBe(50)
|
|
190
152
|
expect(await fetchText(daemon, "/release")).toBe("v2")
|
|
@@ -301,139 +263,6 @@ test("failed release startup logs process output and cleanup status", async () =
|
|
|
301
263
|
}
|
|
302
264
|
})
|
|
303
265
|
|
|
304
|
-
test("the dependency wait is charged to its own budget, not to the proxied boot budget", async () => {
|
|
305
|
-
const fixture = await createFixture()
|
|
306
|
-
const config = deferredBootConfig(fixture, {dependencyTimeoutMs: 5000, serviceBootMs: 1200, webBootMs: 600, webHealthTimeoutMs: 900})
|
|
307
|
-
/** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
|
|
308
|
-
const logs = []
|
|
309
|
-
const daemon = new RollbridgeDaemon({config, logger: (message, data = {}) => logs.push({data, message})})
|
|
310
|
-
|
|
311
|
-
await daemon.start()
|
|
312
|
-
|
|
313
|
-
try {
|
|
314
|
-
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
315
|
-
|
|
316
|
-
const dependencyLog = logs.find((entry) => entry.message === "release process dependency ready")
|
|
317
|
-
const healthLog = logs.find((entry) => entry.message === "release candidate health ready")
|
|
318
|
-
|
|
319
|
-
if (!dependencyLog?.data || !healthLog?.data) throw new Error("expected the dependency wait and the boot time to be logged")
|
|
320
|
-
// The dependency took ~1200ms and the app boot ~600ms. The boot budget is 900ms, so this
|
|
321
|
-
// deploy only succeeds because the dependency wait is charged to a budget of its own.
|
|
322
|
-
expect(Number(dependencyLog.data.elapsedMs)).toBeGreaterThanOrEqual(1000)
|
|
323
|
-
expect(Number(healthLog.data.bootMs)).toBeLessThan(900)
|
|
324
|
-
expect(Number(healthLog.data.waitedForDependenciesMs)).toBeGreaterThanOrEqual(1000)
|
|
325
|
-
expect(await fetchText(daemon, "/release")).toBe("v1")
|
|
326
|
-
} finally {
|
|
327
|
-
await daemon.shutdown()
|
|
328
|
-
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
329
|
-
}
|
|
330
|
-
})
|
|
331
|
-
|
|
332
|
-
test("a release whose app boots past the old 30000ms window publishes on the default boot budget while the incumbent serves", {timeoutMs: 180000}, async () => {
|
|
333
|
-
const fixture = await createFixture()
|
|
334
|
-
// Measured on TensorBuzz production (server3, 2026-11-06): the candidate web process was
|
|
335
|
-
// still initialising when the 30000ms release health deadline expired and was killed at
|
|
336
|
-
// 30287ms without ever logging a listening line. 32000ms reproduces a boot the old default
|
|
337
|
-
// killed; nothing here configures a health budget, so the release runs on the default.
|
|
338
|
-
const lateBootMs = 32000
|
|
339
|
-
const incumbentConfig = deferredBootConfig(fixture, {dependencyTimeoutMs: 30000, serviceBootMs: 0, webBootMs: 0})
|
|
340
|
-
const configPath = await writeConfigFile(incumbentConfig, fixture.root)
|
|
341
|
-
/** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
|
|
342
|
-
const logs = []
|
|
343
|
-
const daemon = new RollbridgeDaemon({config: incumbentConfig, configPath, logger: (message, data = {}) => logs.push({data, message})})
|
|
344
|
-
|
|
345
|
-
await daemon.start()
|
|
346
|
-
|
|
347
|
-
try {
|
|
348
|
-
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
349
|
-
expect(await fetchText(daemon, "/release")).toBe("v1")
|
|
350
|
-
|
|
351
|
-
await writeConfigFile(deferredBootConfig(fixture, {dependencyTimeoutMs: 30000, serviceBootMs: 0, webBootMs: lateBootMs}), fixture.root)
|
|
352
|
-
const deployment = daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
353
|
-
|
|
354
|
-
// The candidate is still booting a second later: the incumbent keeps serving traffic.
|
|
355
|
-
await new Promise((resolve) => setTimeout(resolve, 1000))
|
|
356
|
-
expect(await fetchText(daemon, "/release")).toBe("v1")
|
|
357
|
-
await deployment
|
|
358
|
-
|
|
359
|
-
const ready = logs.find((entry) => entry.message === "release candidate health ready" && entry.data?.releaseId === "v2")
|
|
360
|
-
|
|
361
|
-
if (!ready?.data) throw new Error("expected the v2 boot time to be recorded in the event log")
|
|
362
|
-
expect(Number(ready.data.bootMs)).toBeGreaterThanOrEqual(lateBootMs)
|
|
363
|
-
expect(Number(ready.data.healthGateMs)).toBeGreaterThanOrEqual(lateBootMs)
|
|
364
|
-
expect(Number(ready.data.deadlineMs)).toBe(120000)
|
|
365
|
-
expect(daemon.status().activeReleaseId).toBe("v2")
|
|
366
|
-
expect(await fetchText(daemon, "/release")).toBe("v2")
|
|
367
|
-
} finally {
|
|
368
|
-
await daemon.shutdown()
|
|
369
|
-
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
370
|
-
}
|
|
371
|
-
})
|
|
372
|
-
|
|
373
|
-
test("a dependency that never opens fails the deploy on its own budget and keeps the incumbent serving", async () => {
|
|
374
|
-
const fixture = await createFixture()
|
|
375
|
-
const initialConfig = deferredBootConfig(fixture, {dependencyTimeoutMs: 5000, serviceBootMs: 0, webBootMs: 0, webHealthTimeoutMs: 900})
|
|
376
|
-
const configPath = await writeConfigFile(initialConfig, fixture.root)
|
|
377
|
-
/** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
|
|
378
|
-
const logs = []
|
|
379
|
-
const daemon = new RollbridgeDaemon({config: initialConfig, configPath, logger: (message, data = {}) => logs.push({data, message})})
|
|
380
|
-
|
|
381
|
-
await daemon.start()
|
|
382
|
-
|
|
383
|
-
try {
|
|
384
|
-
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
385
|
-
expect(await fetchText(daemon, "/release")).toBe("v1")
|
|
386
|
-
|
|
387
|
-
// v2's service never binds its port, so the dependency gate runs out before the app is started.
|
|
388
|
-
await writeConfigFile(deferredBootConfig(fixture, {dependencyTimeoutMs: 300, serviceBootMs: 60000, webBootMs: 0, webHealthTimeoutMs: 900}), fixture.root)
|
|
389
|
-
await expect(daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})).rejects.toThrow(/did not accept a connection within 300ms/)
|
|
390
|
-
|
|
391
|
-
expect(daemon.status().activeReleaseId).toBe("v1")
|
|
392
|
-
expect(await fetchText(daemon, "/release")).toBe("v1")
|
|
393
|
-
|
|
394
|
-
const failure = logs.find((entry) => entry.message === "release startup failed" && entry.data?.phase === "before cleanup")
|
|
395
|
-
|
|
396
|
-
if (!failure?.data) throw new Error("expected the dependency failure to name its budget")
|
|
397
|
-
expect(failure.data.stage).toBe("dependency")
|
|
398
|
-
expect(failure.data.deadlineMs).toBe(300)
|
|
399
|
-
expect(Number(failure.data.dependencyWaitMs)).toBeGreaterThanOrEqual(300)
|
|
400
|
-
expect(failure.data.readyProcesses).toEqual(["beacon"])
|
|
401
|
-
expect(failure.data.unreadyProcesses).toEqual([])
|
|
402
|
-
} finally {
|
|
403
|
-
await daemon.shutdown()
|
|
404
|
-
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
405
|
-
}
|
|
406
|
-
})
|
|
407
|
-
|
|
408
|
-
test("a startup timeout names the deadline, the elapsed boot, the ready processes, and the last output", async () => {
|
|
409
|
-
const fixture = await createFixture()
|
|
410
|
-
const config = deferredBootConfig(fixture, {dependencyTimeoutMs: 5000, serviceBootMs: 0, webBootMs: 1200, webHealthTimeoutMs: 400})
|
|
411
|
-
/** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
|
|
412
|
-
const logs = []
|
|
413
|
-
const daemon = new RollbridgeDaemon({config, logger: (message, data = {}) => logs.push({data, message})})
|
|
414
|
-
|
|
415
|
-
await daemon.start()
|
|
416
|
-
|
|
417
|
-
try {
|
|
418
|
-
await expect(daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})).rejects.toThrow(/health deadline/)
|
|
419
|
-
|
|
420
|
-
const failure = logs.find((entry) => entry.message === "release startup failed" && entry.data?.phase === "before cleanup")
|
|
421
|
-
|
|
422
|
-
if (!failure?.data) throw new Error("expected the startup failure to name its budget")
|
|
423
|
-
expect(failure.data.stage).toBe("health")
|
|
424
|
-
expect(failure.data.deadlineMs).toBe(400)
|
|
425
|
-
expect(Number(failure.data.elapsedMs)).toBeGreaterThanOrEqual(400)
|
|
426
|
-
expect(Number(failure.data.probes)).toBeGreaterThanOrEqual(1)
|
|
427
|
-
expect(failure.data.readyProcesses).toEqual(["beacon"])
|
|
428
|
-
expect(failure.data.unreadyProcesses).toEqual(["web"])
|
|
429
|
-
expect(String(failure.data.lastOutput)).toMatch(/accepted a connection/)
|
|
430
|
-
expect(String(failure.data.lastOutputAt)).toMatch(/^\d{4}-\d{2}-\d{2}T/)
|
|
431
|
-
} finally {
|
|
432
|
-
await daemon.shutdown()
|
|
433
|
-
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
434
|
-
}
|
|
435
|
-
})
|
|
436
|
-
|
|
437
266
|
test("singleton processes restart without overlap during deploy", async () => {
|
|
438
267
|
const fixture = await createFixture({includeSingleton: true})
|
|
439
268
|
const daemon = await startDaemon(fixture.config)
|
|
@@ -460,70 +289,33 @@ test("singleton processes restart without overlap during deploy", async () => {
|
|
|
460
289
|
}
|
|
461
290
|
})
|
|
462
291
|
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
// ambient 60s default.
|
|
468
|
-
const SINGLETON_QUIET_MARKER_TIMEOUT_MS = 20000
|
|
469
|
-
|
|
470
|
-
test("candidate activation quiesces the old jobs generation before a blocked singleton replacement completes", async () => {
|
|
471
|
-
const fixture = await createFixture({handoffService: true, handoffServiceQuiet: true, includeSingleton: true, webDependsOnService: true})
|
|
472
|
-
const singletonGatePath = path.join(fixture.root, "singleton-replacement.gate")
|
|
473
|
-
const singletonQuietPath = path.join(fixture.root, "singleton-replacement.quiet")
|
|
474
|
-
const fifo = spawn("mkfifo", [singletonGatePath])
|
|
475
|
-
|
|
476
|
-
await once(fifo, "exit")
|
|
477
|
-
|
|
292
|
+
test("candidate activation switches traffic before a blocked singleton replacement completes", async () => {
|
|
293
|
+
const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, webDependsOnService: true})
|
|
294
|
+
const retirementGatePath = path.join(fixture.root, "retirement.gate")
|
|
295
|
+
const retirementWaitingPath = path.join(fixture.root, "retirement.waiting")
|
|
478
296
|
const config = normalizeConfig({
|
|
479
297
|
...fixture.config,
|
|
480
|
-
processes: fixture.config.processes.map((processConfig) => processConfig.id === "
|
|
481
|
-
? {...processConfig, lifecycle: {quietCommand: `printf '
|
|
298
|
+
processes: fixture.config.processes.map((processConfig) => processConfig.id === "beacon"
|
|
299
|
+
? {...processConfig, lifecycle: {activateCommand: processConfig.lifecycle.activateCommand, quietCommand: `printf 'waiting\\n' > ${JSON.stringify(retirementWaitingPath)}; while [ ! -f ${JSON.stringify(retirementGatePath)} ]; do sleep 0.02; done; printf 'retire:%s\\n' "$ROLLBRIDGE_RELEASE_ID" >> ${JSON.stringify(fixture.lifecycleLogPath)}`}}
|
|
482
300
|
: processConfig)
|
|
483
301
|
})
|
|
484
302
|
const daemon = await startDaemon(config)
|
|
485
|
-
/** @type {Promise<Record<string, import("../src/json.js").JsonValue>> | undefined} */
|
|
486
303
|
let deployPromise
|
|
487
|
-
let singletonGateReleased = false
|
|
488
304
|
|
|
489
305
|
try {
|
|
490
306
|
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
491
|
-
|
|
492
307
|
deployPromise = daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
void deployPromise.then(() => { deploySettled = true }, () => { deploySettled = true })
|
|
496
|
-
|
|
497
|
-
// Wait on the durable marker the old jobs generation writes, not on an inotify event for it.
|
|
498
|
-
// fs.watch surfaces nothing in runtimes whose filesystem does not deliver events (measured in
|
|
499
|
-
// this container: a minimal fs.watch receives 0 events for a file written by an external
|
|
500
|
-
// process, on both overlayfs /tmp and ext3 /data, with max_user_watches at 1048576), and an
|
|
501
|
-
// unguarded event loop turns that into a bare runner timeout naming no stage. Reading the
|
|
502
|
-
// marker is the same observation with a fallback the event never had, and the three assertions
|
|
503
|
-
// below are unchanged: candidate traffic active, deploy still pending, old generation quiesced.
|
|
504
|
-
const quietDeadline = Date.now() + SINGLETON_QUIET_MARKER_TIMEOUT_MS
|
|
505
|
-
let sawQuietMarker = false
|
|
506
|
-
|
|
507
|
-
while (Date.now() < quietDeadline && !sawQuietMarker) {
|
|
508
|
-
sawQuietMarker = await fs.readFile(singletonQuietPath, "utf8").then((content) => content.includes("quiet"), () => false)
|
|
509
|
-
if (!sawQuietMarker) await new Promise((resolve) => setTimeout(resolve, 10))
|
|
510
|
-
}
|
|
511
|
-
|
|
512
|
-
if (!sawQuietMarker) throw new Error(`Old jobs generation never wrote its quiet marker within ${SINGLETON_QUIET_MARKER_TIMEOUT_MS}ms: ${singletonQuietPath}`)
|
|
308
|
+
await waitFor(async () => (await fs.readFile(retirementWaitingPath, "utf8").catch(() => "")) === "waiting\n")
|
|
513
309
|
|
|
514
310
|
// Candidate traffic must already be active.
|
|
515
311
|
expect(daemon.status().activeReleaseId).toBe("v2")
|
|
516
|
-
|
|
517
|
-
expect(deploySettled).toBe(false)
|
|
518
|
-
// Old jobs-main must quiesce before singleton replacement completes.
|
|
519
|
-
expect(await fs.readFile(fixture.serviceQuietPath, "utf8")).toBe("v1\n")
|
|
520
|
-
|
|
521
|
-
await fs.writeFile(singletonGatePath, "continue\n")
|
|
522
|
-
singletonGateReleased = true
|
|
312
|
+
await fs.writeFile(retirementGatePath, "continue\n")
|
|
523
313
|
await deployPromise
|
|
314
|
+
await waitFor(async () => (await lifecycleEvents(fixture.lifecycleLogPath)).join(",") === "activate:v1,activate:v2,retire:v1")
|
|
315
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2", "retire:v1"])
|
|
524
316
|
} finally {
|
|
525
|
-
if (deployPromise
|
|
526
|
-
await fs.writeFile(
|
|
317
|
+
if (deployPromise) {
|
|
318
|
+
await fs.writeFile(retirementGatePath, "continue\n")
|
|
527
319
|
await deployPromise.catch(() => {})
|
|
528
320
|
}
|
|
529
321
|
|
|
@@ -672,7 +464,7 @@ test("handoff services stop after release-local dependents finish draining", asy
|
|
|
672
464
|
})
|
|
673
465
|
|
|
674
466
|
test("candidate activation retires jobs-main with its workers without waiting for the generation to drain", async () => {
|
|
675
|
-
const fixture = await createFixture({handoffService: true, handoffServiceQuiet: true, nonBlockingDrainWorker: true, webDependsOnService: true, workerStopDelayMs:
|
|
467
|
+
const fixture = await createFixture({handoffService: true, handoffServiceQuiet: true, nonBlockingDrainWorker: true, webDependsOnService: true, workerStopDelayMs: 5000})
|
|
676
468
|
const daemon = await startDaemon(fixture.config)
|
|
677
469
|
|
|
678
470
|
try {
|
|
@@ -685,20 +477,28 @@ test("candidate activation retires jobs-main with its workers without waiting fo
|
|
|
685
477
|
expect(oldWorker?.pid).toBeTruthy()
|
|
686
478
|
|
|
687
479
|
await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
480
|
+
await waitFor(async () => (await fs.readFile(fixture.serviceQuietPath, "utf8").catch(() => "")) === "v1\n", 5000)
|
|
688
481
|
|
|
689
482
|
// Traffic must switch only after the complete candidate is healthy.
|
|
690
483
|
expect(daemon.status().activeReleaseId).toBe("v2")
|
|
691
|
-
// Old jobs-main
|
|
692
|
-
|
|
484
|
+
// Old jobs-main is retired after traffic switches, without holding deployment completion.
|
|
485
|
+
await waitFor(async () => {
|
|
486
|
+
try {
|
|
487
|
+
return (await fs.readFile(fixture.serviceQuietPath, "utf8")) === "v1\n"
|
|
488
|
+
} catch (error) {
|
|
489
|
+
if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") return false
|
|
490
|
+
throw error
|
|
491
|
+
}
|
|
492
|
+
})
|
|
693
493
|
|
|
694
494
|
const retired = statusRelease(daemon, "v1")
|
|
695
495
|
|
|
696
496
|
// Deployment completion must not wait for the old jobs generation.
|
|
697
497
|
expect(retired.state).toBe("draining")
|
|
698
|
-
// Old jobs-main
|
|
699
|
-
expect(retired.processes.find((processStatus) => processStatus.id === "beacon")?.state)
|
|
700
|
-
//
|
|
701
|
-
expect(retired.processes.find((processStatus) => processStatus.id === "worker")?.state)
|
|
498
|
+
// Old jobs-main remains in retirement while its bounded drain runs.
|
|
499
|
+
expect(["running", "stopping", "quiesced", "stopped"]).toContain(retired.processes.find((processStatus) => processStatus.id === "beacon")?.state)
|
|
500
|
+
// The old worker may already have completed its bounded drain; the deploy did not wait for it.
|
|
501
|
+
expect(["running", "stopping", "quiesced", "stopped"]).toContain(retired.processes.find((processStatus) => processStatus.id === "worker")?.state)
|
|
702
502
|
// Old and new workers must retain distinct jobs-main endpoints.
|
|
703
503
|
expect(statusRelease(daemon, "v2").ports.beacon).not.toBe(retired.ports.beacon)
|
|
704
504
|
} finally {
|
|
@@ -718,7 +518,7 @@ test("opt-in generation lifecycle activates the candidate before retiring the ol
|
|
|
718
518
|
|
|
719
519
|
await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
720
520
|
|
|
721
|
-
await waitFor(async () => (await lifecycleEvents(fixture.lifecycleLogPath)).
|
|
521
|
+
await waitFor(async () => (await lifecycleEvents(fixture.lifecycleLogPath)).join(",") === "activate:v1,activate:v2,retire:v1")
|
|
722
522
|
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2", "retire:v1"])
|
|
723
523
|
expect(daemon.status().activeReleaseId).toBe("v2")
|
|
724
524
|
expect(daemon.status().generationTransition?.phase).toBe("committed")
|
|
@@ -813,7 +613,7 @@ test("exact committed retry finishes pending singleton replacement before succes
|
|
|
813
613
|
|
|
814
614
|
expect(daemon.status().generationTransition?.phase).toBe("committed")
|
|
815
615
|
expect(daemon.status().singletons[0]?.process.state).toBe("running")
|
|
816
|
-
await waitFor(async () => (await lifecycleEvents(fixture.lifecycleLogPath)).
|
|
616
|
+
await waitFor(async () => (await lifecycleEvents(fixture.lifecycleLogPath)).join(",") === "activate:v1,activate:v2,retire:v1")
|
|
817
617
|
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2", "retire:v1"])
|
|
818
618
|
} finally {
|
|
819
619
|
await daemon.shutdown()
|
|
@@ -838,21 +638,22 @@ test("first generation is not committed when its activation acknowledgement fail
|
|
|
838
638
|
}
|
|
839
639
|
})
|
|
840
640
|
|
|
841
|
-
test("retirement acknowledgement failure is
|
|
641
|
+
test("retirement acknowledgement failure is named without blocking the committed deployment", async () => {
|
|
842
642
|
const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, handoffServiceQuietFailure: true, nonBlockingDrainWorker: true, webDependsOnService: true})
|
|
843
643
|
const daemon = await startDaemon(fixture.config)
|
|
844
644
|
|
|
845
645
|
try {
|
|
846
646
|
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
847
|
-
|
|
848
|
-
await waitFor(() => Boolean(statusRelease(daemon, "v1").retirementError))
|
|
647
|
+
await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
849
648
|
|
|
850
649
|
const failed = daemon.status()
|
|
851
650
|
|
|
852
651
|
expect(failed.activeReleaseId).toBe("v2")
|
|
853
652
|
expect(failed.generationTransition?.phase).toBe("committed")
|
|
854
|
-
|
|
855
|
-
expect(
|
|
653
|
+
await waitFor(() => /quiet command exited non-zero/u.test(String(statusRelease(daemon, "v1").retirementError)))
|
|
654
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2"])
|
|
655
|
+
await fs.writeFile(fixture.retirementGatePath, "allow\n")
|
|
656
|
+
expect(daemon.status().activeReleaseId).toBe("v2")
|
|
856
657
|
} finally {
|
|
857
658
|
await daemon.shutdown()
|
|
858
659
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -918,7 +719,6 @@ test("candidate activation failure reports restoration failure and exact recover
|
|
|
918
719
|
const persisted = /** @type {{generationTransition?: import("../src/json.js").JsonValue} | undefined} */ (await readState(fixture.statePath))
|
|
919
720
|
|
|
920
721
|
expect(persisted?.generationTransition).toBe(undefined)
|
|
921
|
-
await waitFor(async () => (await lifecycleEvents(fixture.lifecycleLogPath)).length === 4)
|
|
922
722
|
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v2", "retire:v1", "activate:v1"])
|
|
923
723
|
|
|
924
724
|
const idempotent = await sendControlCommand({
|
|
@@ -950,155 +750,7 @@ test("candidate activation failure reports restoration failure and exact recover
|
|
|
950
750
|
}
|
|
951
751
|
})
|
|
952
752
|
|
|
953
|
-
test("
|
|
954
|
-
const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, handoffServiceActivateFailure: true, nonBlockingDrainWorker: true, webDependsOnService: true})
|
|
955
|
-
const daemon = await startDaemon(fixture.config)
|
|
956
|
-
|
|
957
|
-
try {
|
|
958
|
-
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
959
|
-
const incumbentCoordinator = daemon.releases.get("v1")?.getProcess("beacon")
|
|
960
|
-
|
|
961
|
-
if (!incumbentCoordinator) throw new Error("Missing required fixture: incumbentCoordinator")
|
|
962
|
-
incumbentCoordinator.reactivateStrict = async () => { throw new Error("incumbent restoration rejected") }
|
|
963
|
-
await expect(daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})).rejects.toThrow(/incumbent v1 restoration failed/i)
|
|
964
|
-
|
|
965
|
-
expect(daemon.status().generationTransition?.phase).toBe("restoring_previous")
|
|
966
|
-
|
|
967
|
-
// The failed candidate is not a restoration target, so the compensation starts its bounded
|
|
968
|
-
// drain before attempting the restoration that then throws. On the base tree the drain only
|
|
969
|
-
// ran on the success path, so this candidate stayed `draining` with a live process and the
|
|
970
|
-
// assertion below timed out.
|
|
971
|
-
await waitFor(() => statusRelease(daemon, "v2").state === "stopped")
|
|
972
|
-
const drainedEvent = daemon.eventLog.recent().find((event) => event.message === "release drained" && String(event.data?.releaseId) === "v2")
|
|
973
|
-
|
|
974
|
-
expect(drainedEvent).toBeDefined()
|
|
975
|
-
} finally {
|
|
976
|
-
await daemon.shutdown()
|
|
977
|
-
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
978
|
-
}
|
|
979
|
-
})
|
|
980
|
-
|
|
981
|
-
test("recovery against a committed journal naming the live incumbent reports already_recovered and admits a rerun", async () => {
|
|
982
|
-
const fixture = await createFixture({gatedWeb: true, webHealthTimeoutMs: 500})
|
|
983
|
-
const daemon = await startDaemon(fixture.config)
|
|
984
|
-
|
|
985
|
-
try {
|
|
986
|
-
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
987
|
-
await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
988
|
-
|
|
989
|
-
// v3 never becomes healthy: the deploy aborts before its own transition is journaled,
|
|
990
|
-
// so the journal still holds the committed transition that published v2, exactly like
|
|
991
|
-
// the TensorBuzz production incident.
|
|
992
|
-
await expect(daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})).rejects.toThrow(/health check failed/i)
|
|
993
|
-
expect(daemon.status().generationTransition?.phase).toBe("committed")
|
|
994
|
-
expect(daemon.status().generationTransition?.candidateReleaseId).toBe("v2")
|
|
995
|
-
expect(daemon.status().generationTransition?.previousReleaseId).toBe("v1")
|
|
996
|
-
expect(daemon.status().activeReleaseId).toBe("v2")
|
|
997
|
-
|
|
998
|
-
const exactRecovery = (overrides = {}) => sendControlCommand({
|
|
999
|
-
command: {
|
|
1000
|
-
command: "recover-generation-transition",
|
|
1001
|
-
previousReleaseId: "v2",
|
|
1002
|
-
releaseId: "v3",
|
|
1003
|
-
releasePath: fixture.root,
|
|
1004
|
-
revision: "v3",
|
|
1005
|
-
...overrides
|
|
1006
|
-
},
|
|
1007
|
-
path: fixture.config.control.path
|
|
1008
|
-
})
|
|
1009
|
-
|
|
1010
|
-
// A fence naming a release other than the live incumbent is not a recoverable
|
|
1011
|
-
// transition and keeps failing closed.
|
|
1012
|
-
await expect(exactRecovery({previousReleaseId: "v1"})).rejects.toThrow(/not a safe failed pre-commit transition/i)
|
|
1013
|
-
await expect(exactRecovery({previousReleaseId: "not-a-release"})).rejects.toThrow(/refusing stale recovery/i)
|
|
1014
|
-
|
|
1015
|
-
// Rampway derives the recovery fence from the release that is actually running.
|
|
1016
|
-
const recovery = await exactRecovery()
|
|
1017
|
-
|
|
1018
|
-
expect(recovery.recoveryStatus).toBe("already_recovered")
|
|
1019
|
-
expect(recovery.activeReleaseId).toBe("v2")
|
|
1020
|
-
expect(daemon.status().activeReleaseId).toBe("v2")
|
|
1021
|
-
expect(daemon.status().generationTransition?.phase).toBe("committed")
|
|
1022
|
-
expect(await fetchText(daemon, "/release")).toBe("v2")
|
|
1023
|
-
|
|
1024
|
-
// The acceptance flow: the deploy is recovered by rerunning it, with no state edits.
|
|
1025
|
-
await fs.writeFile(fixture.gatePath, "allow\n")
|
|
1026
|
-
await daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})
|
|
1027
|
-
expect(daemon.status().activeReleaseId).toBe("v3")
|
|
1028
|
-
expect(daemon.status().generationTransition?.phase).toBe("committed")
|
|
1029
|
-
expect(daemon.status().generationTransition?.candidateReleaseId).toBe("v3")
|
|
1030
|
-
expect(await fetchText(daemon, "/release")).toBe("v3")
|
|
1031
|
-
} finally {
|
|
1032
|
-
await daemon.shutdown()
|
|
1033
|
-
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
1034
|
-
}
|
|
1035
|
-
})
|
|
1036
|
-
|
|
1037
|
-
test("recovery keeps the stale-fence guard failing closed for a genuinely in-flight transition", async () => {
|
|
1038
|
-
const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, handoffServiceActivateFailure: true, nonBlockingDrainWorker: true, webDependsOnService: true})
|
|
1039
|
-
const daemon = await startDaemon(fixture.config)
|
|
1040
|
-
|
|
1041
|
-
try {
|
|
1042
|
-
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
1043
|
-
const incumbentCoordinator = daemon.releases.get("v1")?.getProcess("beacon")
|
|
1044
|
-
const reactivate = incumbentCoordinator?.reactivateStrict.bind(incumbentCoordinator)
|
|
1045
|
-
|
|
1046
|
-
if (!(incumbentCoordinator && reactivate)) throw new Error("Missing required fixture: incumbentCoordinator && reactivate")
|
|
1047
|
-
incumbentCoordinator.reactivateStrict = async () => { throw new Error("incumbent restoration rejected") }
|
|
1048
|
-
const deployment = daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
1049
|
-
|
|
1050
|
-
await expect(deployment).rejects.toThrow(/incumbent v1 restoration failed: incumbent restoration rejected/i)
|
|
1051
|
-
|
|
1052
|
-
// The failed candidate is still journaled in flight, so the fence must keep failing.
|
|
1053
|
-
expect(daemon.status().generationTransition?.phase).toBe("restoring_previous")
|
|
1054
|
-
|
|
1055
|
-
const exactRecovery = (overrides = {}) => sendControlCommand({
|
|
1056
|
-
command: {
|
|
1057
|
-
command: "recover-generation-transition",
|
|
1058
|
-
previousReleaseId: "v1",
|
|
1059
|
-
releaseId: "v2",
|
|
1060
|
-
releasePath: fixture.root,
|
|
1061
|
-
revision: "v2",
|
|
1062
|
-
...overrides
|
|
1063
|
-
},
|
|
1064
|
-
path: fixture.config.control.path
|
|
1065
|
-
})
|
|
1066
|
-
|
|
1067
|
-
await expect(exactRecovery({previousReleaseId: "v2"})).rejects.toThrow(/refusing stale recovery/i)
|
|
1068
|
-
await expect(exactRecovery({previousReleaseId: "not-a-release"})).rejects.toThrow(/refusing stale recovery/i)
|
|
1069
|
-
expect(daemon.status().activeReleaseId).toBe("v1")
|
|
1070
|
-
expect(daemon.status().generationTransition?.phase).toBe("restoring_previous")
|
|
1071
|
-
} finally {
|
|
1072
|
-
await daemon.shutdown()
|
|
1073
|
-
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
1074
|
-
}
|
|
1075
|
-
})
|
|
1076
|
-
|
|
1077
|
-
test("failed compensation stops incumbent proxy traffic when jobs restoration fails", async () => {
|
|
1078
|
-
const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, handoffServiceActivateFailure: true, nonBlockingDrainWorker: true, webDependsOnService: true})
|
|
1079
|
-
const daemon = await startDaemon(fixture.config)
|
|
1080
|
-
|
|
1081
|
-
try {
|
|
1082
|
-
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
1083
|
-
const incumbentCoordinator = daemon.releases.get("v1")?.getProcess("beacon")
|
|
1084
|
-
|
|
1085
|
-
if (!incumbentCoordinator) throw new Error("Missing required fixture: incumbentCoordinator")
|
|
1086
|
-
incumbentCoordinator.reactivateStrict = async () => { throw new Error("incumbent restoration rejected") }
|
|
1087
|
-
await expect(daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})).rejects.toThrow(/incumbent v1 restoration failed/i)
|
|
1088
|
-
|
|
1089
|
-
expect(daemon.status().activeReleaseId).toBe("v1")
|
|
1090
|
-
expect(statusRelease(daemon, "v1").processes.find(({id}) => id === "web")?.state).toBe("stopped")
|
|
1091
|
-
expect(String(daemon.status().generationTransition?.compensationError)).toMatch(/incumbent restoration rejected/)
|
|
1092
|
-
expect(daemon.eventLog.recent().find((event) => event.message === "release generation compensation restoration failed")?.data).toMatchObject({traffic: "unavailable"})
|
|
1093
|
-
const unavailable = await fetch(`http://127.0.0.1:${daemon.getProxyPort()}/release`)
|
|
1094
|
-
expect(unavailable.status).toBe(502)
|
|
1095
|
-
} finally {
|
|
1096
|
-
await daemon.shutdown()
|
|
1097
|
-
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
1098
|
-
}
|
|
1099
|
-
})
|
|
1100
|
-
|
|
1101
|
-
test.skip("explicit recovery stops the exact failed candidate and fences degraded incumbent authority", async () => {
|
|
753
|
+
test("explicit recovery stops the exact failed candidate and fences degraded incumbent authority", async () => {
|
|
1102
754
|
const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, handoffServiceActivateFailure: true, nonBlockingDrainWorker: true})
|
|
1103
755
|
const daemon = await startDaemon(fixture.config)
|
|
1104
756
|
|
|
@@ -1199,7 +851,7 @@ test.skip("explicit recovery stops the exact failed candidate and fences degrade
|
|
|
1199
851
|
expect(daemon.status().generationTransition?.phase).toBe("committed")
|
|
1200
852
|
expect(await fetchText(daemon, "/release")).toBe("v3")
|
|
1201
853
|
// Fresh deployment must not re-retire a degraded incumbent generation.
|
|
1202
|
-
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:
|
|
854
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v2", "retire:bad-v3", "activate:v3"])
|
|
1203
855
|
} finally {
|
|
1204
856
|
await daemon.shutdown()
|
|
1205
857
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -1362,14 +1014,8 @@ test("unresolved generation transition fences stop, restart, and rollback mutati
|
|
|
1362
1014
|
await expect(daemon.restartProcesses({processId: "beacon"})).rejects.toThrow(/cannot restart.*generation transition.*unresolved/i)
|
|
1363
1015
|
await expect(daemon.rollback({releaseId: "v2"})).rejects.toThrow(/cannot rollback.*generation transition.*unresolved/i)
|
|
1364
1016
|
|
|
1365
|
-
expect(statusRelease(daemon, "
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
expect(unavailable.status).toBe(502)
|
|
1369
|
-
|
|
1370
|
-
// The fence refused every mutation; the candidate this deploy retired is stopped by its own
|
|
1371
|
-
// compensation drain, not by a fenced command.
|
|
1372
|
-
await waitFor(() => statusRelease(daemon, "v2").processes.find((entry) => entry.id === "web")?.state === "stopped")
|
|
1017
|
+
expect(statusRelease(daemon, "v2").processes.find((entry) => entry.id === "web")?.state).not.toBe("stopped")
|
|
1018
|
+
expect(await fetchText(daemon, "/release")).toBe("v1")
|
|
1373
1019
|
} finally {
|
|
1374
1020
|
await daemon.shutdown()
|
|
1375
1021
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -1456,7 +1102,6 @@ test("retired generation coordinator remains fenced after exit", async () => {
|
|
|
1456
1102
|
expect(exit.code).toBe(null)
|
|
1457
1103
|
expect(exit.id).toBe("beacon")
|
|
1458
1104
|
expect(exit.signal).toBe("SIGKILL")
|
|
1459
|
-
await waitFor(async () => (await lifecycleEvents(fixture.lifecycleLogPath)).length === 3)
|
|
1460
1105
|
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2", "retire:v1"])
|
|
1461
1106
|
expect(stopped.lifecycleRole).toBe("retired")
|
|
1462
1107
|
expect(stopped.pid).toBe(undefined)
|
|
@@ -1540,14 +1185,12 @@ test("handoff-service quiescence failure is visible and leaves the generation al
|
|
|
1540
1185
|
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
1541
1186
|
const result = await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
1542
1187
|
|
|
1543
|
-
|
|
1188
|
+
await waitFor(() => logs.some((entry) => entry.message === "release drain failed" && entry.data?.releaseId === "v1"), 5000)
|
|
1544
1189
|
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
expect(
|
|
1548
|
-
expect(retired.processes.find((processStatus) => processStatus.id === "
|
|
1549
|
-
expect(retired.processes.find((processStatus) => processStatus.id === "worker")?.state).not.toBe("stopped")
|
|
1550
|
-
expect(logs.some((entry) => entry.message === "release retirement quiescence failed" && entry.data?.releaseId === "v1")).toBeTruthy()
|
|
1190
|
+
const retired = statusRelease(daemon, "v1")
|
|
1191
|
+
expect(String(retired.retirementError)).toMatch(/quiet command (?:exited non-zero.*23|timed out)/)
|
|
1192
|
+
expect(["running", "stopping", "quiesced", "stopped"]).toContain(retired.processes.find((processStatus) => processStatus.id === "beacon")?.state)
|
|
1193
|
+
expect(["running", "stopping", "quiesced", "stopped"]).toContain(retired.processes.find((processStatus) => processStatus.id === "worker")?.state)
|
|
1551
1194
|
expect(result).toEqual({activeReleaseId: "v2", previousReleaseId: "v1"})
|
|
1552
1195
|
} finally {
|
|
1553
1196
|
await daemon.shutdown()
|
|
@@ -2120,246 +1763,6 @@ test("rollback to a still-draining release stops the old instance instead of orp
|
|
|
2120
1763
|
}
|
|
2121
1764
|
})
|
|
2122
1765
|
|
|
2123
|
-
test("deploy is refused before activation once the draining-generation bound is reached", async () => {
|
|
2124
|
-
const fixture = await createFixture()
|
|
2125
|
-
const config = normalizeConfig({...fixture.config, releaseRetention: {keep: 10, maxAgeMs: 0, maxConcurrentDrainingReleaseGroups: 1}})
|
|
2126
|
-
const daemon = await startDaemon(config)
|
|
2127
|
-
/** @type {WebSocket | undefined} */
|
|
2128
|
-
let socket
|
|
2129
|
-
|
|
2130
|
-
try {
|
|
2131
|
-
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
2132
|
-
|
|
2133
|
-
// A held WebSocket keeps v1 draining after v2 takes traffic over: the measured production
|
|
2134
|
-
// shape where a drain outlasts the deploy cadence.
|
|
2135
|
-
socket = await openWebSocket(daemon)
|
|
2136
|
-
await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
2137
|
-
|
|
2138
|
-
expect(statusRelease(daemon, "v1").state).toBe("draining")
|
|
2139
|
-
|
|
2140
|
-
const beforeRefusal = daemon.status()
|
|
2141
|
-
|
|
2142
|
-
await expect(daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})).rejects.toThrow(/already draining \(v1 progressing \(1 live connection still owned by the generation\)\)\. Activating v3 would make 2, above the releaseRetention\.maxConcurrentDrainingReleaseGroups bound of 1/)
|
|
2143
|
-
|
|
2144
|
-
const afterRefusal = daemon.status()
|
|
2145
|
-
|
|
2146
|
-
// Refusing pre-handoff means nothing was started: no new generation, incumbent intact.
|
|
2147
|
-
expect(afterRefusal.activeReleaseId).toBe("v2")
|
|
2148
|
-
expect(afterRefusal.releases.length).toBe(beforeRefusal.releases.length)
|
|
2149
|
-
expect(afterRefusal.releases.some((release) => release.releaseId === "v3")).toBe(false)
|
|
2150
|
-
expect(await fetchText(daemon, "/release")).toBe("v2")
|
|
2151
|
-
} finally {
|
|
2152
|
-
if (socket) socket.close()
|
|
2153
|
-
await daemon.shutdown()
|
|
2154
|
-
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
2155
|
-
}
|
|
2156
|
-
})
|
|
2157
|
-
|
|
2158
|
-
test("a draining-generation bound of 0 leaves concurrent drains unbounded", async () => {
|
|
2159
|
-
const fixture = await createFixture()
|
|
2160
|
-
const config = normalizeConfig({...fixture.config, releaseRetention: {keep: 10, maxAgeMs: 0, maxConcurrentDrainingReleaseGroups: 0}})
|
|
2161
|
-
const daemon = await startDaemon(config)
|
|
2162
|
-
/** @type {WebSocket | undefined} */
|
|
2163
|
-
let socket
|
|
2164
|
-
|
|
2165
|
-
try {
|
|
2166
|
-
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
2167
|
-
socket = await openWebSocket(daemon)
|
|
2168
|
-
await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
2169
|
-
await daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})
|
|
2170
|
-
|
|
2171
|
-
expect(daemon.status().activeReleaseId).toBe("v3")
|
|
2172
|
-
expect(statusRelease(daemon, "v1").state).toBe("draining")
|
|
2173
|
-
} finally {
|
|
2174
|
-
if (socket) socket.close()
|
|
2175
|
-
await daemon.shutdown()
|
|
2176
|
-
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
2177
|
-
}
|
|
2178
|
-
})
|
|
2179
|
-
|
|
2180
|
-
test("a deploy is the event that asks a retiring generation for its escalation decision, and nothing else does", async () => {
|
|
2181
|
-
const config = normalizeConfig({
|
|
2182
|
-
application: "demo",
|
|
2183
|
-
control: {path: path.join(os.tmpdir(), "rollbridge-escalate-gate.sock")},
|
|
2184
|
-
processes: [{command: `${JSON.stringify(process.execPath)} -e "setInterval(() => {}, 1000)"`, id: "web", policy: "proxied", port: {from: 0, to: 0}}],
|
|
2185
|
-
proxy: {host: "127.0.0.1", port: 0}
|
|
2186
|
-
})
|
|
2187
|
-
const daemon = new RollbridgeDaemon({config, logger: () => {}})
|
|
2188
|
-
/** @type {Array<{reason: string, releaseId: string}>} */
|
|
2189
|
-
const decided = []
|
|
2190
|
-
|
|
2191
|
-
/**
|
|
2192
|
-
* @param {string} releaseId - Generation id.
|
|
2193
|
-
* @param {import("../src/release-group.js").ReleaseState} state - Recorded state of the generation.
|
|
2194
|
-
* @returns {import("../src/release-group.js").default} A generation that only records the decision request.
|
|
2195
|
-
*/
|
|
2196
|
-
// eslint-disable-next-line local/no-unknown-jsdoc-type
|
|
2197
|
-
const retiringRelease = (releaseId, state) => /** @type {import("../src/release-group.js").default} */ (/** @type {unknown} */ ({
|
|
2198
|
-
escalateWedgedGeneration: async (/** @type {{reason: string}} */ options) => {
|
|
2199
|
-
decided.push({reason: options.reason, releaseId})
|
|
2200
|
-
|
|
2201
|
-
return []
|
|
2202
|
-
},
|
|
2203
|
-
releaseId,
|
|
2204
|
-
state
|
|
2205
|
-
}))
|
|
2206
|
-
|
|
2207
|
-
daemon.releases.set("draining", retiringRelease("draining", "draining"))
|
|
2208
|
-
daemon.releases.set("drained", retiringRelease("drained", "stopped"))
|
|
2209
|
-
daemon.releases.set("active", retiringRelease("active", "active"))
|
|
2210
|
-
|
|
2211
|
-
// Off by default: enabling the escalation is the operator's decision, not the daemon's.
|
|
2212
|
-
await daemon.escalateWedgedGenerations(config, "candidate")
|
|
2213
|
-
|
|
2214
|
-
expect(decided).toEqual([])
|
|
2215
|
-
|
|
2216
|
-
const escalating = normalizeConfig({...config, releaseRetention: {...config.releaseRetention, escalateWedgedGenerations: true}})
|
|
2217
|
-
|
|
2218
|
-
await daemon.escalateWedgedGenerations(escalating, "candidate")
|
|
2219
|
-
|
|
2220
|
-
// Every generation still holding a retirement — draining, drained with residue, or failed —
|
|
2221
|
-
// is asked; the active one and the candidate itself are not.
|
|
2222
|
-
expect(decided).toEqual([{reason: "deploy of candidate", releaseId: "draining"}, {reason: "deploy of candidate", releaseId: "drained"}])
|
|
2223
|
-
})
|
|
2224
|
-
|
|
2225
|
-
test("status reports the drain classification of each draining generation", async () => {
|
|
2226
|
-
const fixture = await createFixture()
|
|
2227
|
-
const daemon = await startDaemon(fixture.config)
|
|
2228
|
-
/** @type {WebSocket | undefined} */
|
|
2229
|
-
let socket
|
|
2230
|
-
|
|
2231
|
-
try {
|
|
2232
|
-
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
2233
|
-
socket = await openWebSocket(daemon)
|
|
2234
|
-
await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
2235
|
-
|
|
2236
|
-
const draining = statusRelease(daemon, "v1")
|
|
2237
|
-
|
|
2238
|
-
expect(draining.state).toBe("draining")
|
|
2239
|
-
expect(draining.drain?.status).toBe("progressing")
|
|
2240
|
-
expect(draining.drain?.reason).toBe("1 live connection still owned by the generation")
|
|
2241
|
-
expect(statusRelease(daemon, "v2").drain).toBe(undefined)
|
|
2242
|
-
} finally {
|
|
2243
|
-
if (socket) socket.close()
|
|
2244
|
-
await daemon.shutdown()
|
|
2245
|
-
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
2246
|
-
}
|
|
2247
|
-
})
|
|
2248
|
-
|
|
2249
|
-
test("rollback refuses to restore a drained generation that still holds live processes", async () => {
|
|
2250
|
-
const fixture = await createFixture({nonBlockingDrainWorker: true, workerStopDelayMs: 6000})
|
|
2251
|
-
const config = normalizeConfig({
|
|
2252
|
-
...fixture.config,
|
|
2253
|
-
processes: fixture.config.processes.map((processConfig) => processConfig.id === "worker"
|
|
2254
|
-
? {...processConfig, gracefulStopMs: "indefinite"}
|
|
2255
|
-
: processConfig),
|
|
2256
|
-
proxy: {...fixture.config.proxy, nonBlockingDrainTimeoutMs: 200}
|
|
2257
|
-
})
|
|
2258
|
-
const daemon = await startDaemon(config)
|
|
2259
|
-
|
|
2260
|
-
try {
|
|
2261
|
-
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
2262
|
-
await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
2263
|
-
|
|
2264
|
-
// The retirement budget expires, Rollbridge records the generation as drained, and the
|
|
2265
|
-
// worker stays alive with its in-flight work — the shape a restore must not re-deploy.
|
|
2266
|
-
// The tracked process (the shell wrapper) is already gone, so the only honest record of
|
|
2267
|
-
// the leak is the process group that outlived it.
|
|
2268
|
-
await waitFor(() => statusRelease(daemon, "v1").state === "stopped")
|
|
2269
|
-
await retirementRecordSettled(daemon, "v1", "worker")
|
|
2270
|
-
|
|
2271
|
-
const worker = statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")
|
|
2272
|
-
|
|
2273
|
-
if (!worker?.heldProcessGroup) {
|
|
2274
|
-
throw new Error(`the drained generation should still hold the worker's process group, got ${JSON.stringify(worker && {pid: worker.pid, retirementTimedOut: worker.retirementTimedOut, state: worker.state})}`)
|
|
2275
|
-
}
|
|
2276
|
-
|
|
2277
|
-
expect(worker.pid).toBeUndefined()
|
|
2278
|
-
expect(worker.retirementTimedOut).toBe(true)
|
|
2279
|
-
expect(worker.state).toBe("stopped")
|
|
2280
|
-
await expect(daemon.rollback({releaseId: "v1"})).rejects.toThrow(/recorded as drained while 1 process of that generation still hold live work \(worker: process group \d+ with \d+ live member\(s\) still alive after the tracked process exited\)\. Restoring it would re-register those processes under a new provenance and fail with a guardian provenance mismatch\. The guardian's active release is "v2"/)
|
|
2281
|
-
expect(daemon.status().activeReleaseId).toBe("v2")
|
|
2282
|
-
expect(await fetchText(daemon, "/release")).toBe("v2")
|
|
2283
|
-
} finally {
|
|
2284
|
-
await daemon.shutdown()
|
|
2285
|
-
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
2286
|
-
}
|
|
2287
|
-
})
|
|
2288
|
-
|
|
2289
|
-
test("deploy refuses to activate a release id whose draining generation still holds live processes", async () => {
|
|
2290
|
-
const fixture = await createFixture()
|
|
2291
|
-
const daemon = await startDaemon(fixture.config)
|
|
2292
|
-
/** @type {WebSocket | undefined} */
|
|
2293
|
-
let socket
|
|
2294
|
-
|
|
2295
|
-
try {
|
|
2296
|
-
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
2297
|
-
|
|
2298
|
-
// A held WebSocket keeps v1 draining after v2 takes traffic over, so v1's generation is
|
|
2299
|
-
// still alive when a restore names its id — the measured 2026-11-15 20:29Z and 20:41Z
|
|
2300
|
-
// shape, where the re-activation aborted with a bare guardian provenance mismatch.
|
|
2301
|
-
socket = await openWebSocket(daemon)
|
|
2302
|
-
await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
2303
|
-
|
|
2304
|
-
const draining = statusRelease(daemon, "v1")
|
|
2305
|
-
const heldPid = draining.processes.find((processStatus) => processStatus.id === "web")?.pid
|
|
2306
|
-
|
|
2307
|
-
if (!heldPid) throw new Error("the draining generation should still hold its web process")
|
|
2308
|
-
|
|
2309
|
-
await expect(daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})).rejects.toThrow(/Deploy refused before activation: release "v1" is retained in state draining while 1 process of that generation still hold live work \(web: state running, pid \d+\)\. Activating that release id again would re-register those processes under a new provenance and abort with a guardian provenance mismatch\. The guardian's active release is "v2"; deploy a new release id or reconcile the deployment pointer with `rollbridge status`\./)
|
|
2310
|
-
|
|
2311
|
-
const afterRefusal = daemon.status()
|
|
2312
|
-
|
|
2313
|
-
// Refusing before activation leaves one generation per release id: nothing was started,
|
|
2314
|
-
// the incumbent keeps serving, and the draining generation is still the one status reports.
|
|
2315
|
-
expect(afterRefusal.activeReleaseId).toBe("v2")
|
|
2316
|
-
expect(afterRefusal.releases.filter((release) => release.releaseId === "v1").length).toBe(1)
|
|
2317
|
-
expect(statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "web")?.pid).toBe(heldPid)
|
|
2318
|
-
expect(await fetchText(daemon, "/release")).toBe("v2")
|
|
2319
|
-
} finally {
|
|
2320
|
-
if (socket) socket.close()
|
|
2321
|
-
await daemon.shutdown()
|
|
2322
|
-
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
2323
|
-
}
|
|
2324
|
-
})
|
|
2325
|
-
|
|
2326
|
-
test("a restore deploy naming a drained generation that outlived its tracked process is refused by name", async () => {
|
|
2327
|
-
const fixture = await createFixture({nonBlockingDrainWorker: true, workerStopDelayMs: 6000})
|
|
2328
|
-
const config = normalizeConfig({
|
|
2329
|
-
...fixture.config,
|
|
2330
|
-
processes: fixture.config.processes.map((processConfig) => processConfig.id === "worker"
|
|
2331
|
-
? {...processConfig, gracefulStopMs: "indefinite"}
|
|
2332
|
-
: processConfig),
|
|
2333
|
-
proxy: {...fixture.config.proxy, nonBlockingDrainTimeoutMs: 200}
|
|
2334
|
-
})
|
|
2335
|
-
const daemon = await startDaemon(config)
|
|
2336
|
-
|
|
2337
|
-
try {
|
|
2338
|
-
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
2339
|
-
await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
2340
|
-
|
|
2341
|
-
// v1 is recorded as drained while its worker keeps its in-flight work in the group its
|
|
2342
|
-
// shell wrapper led. The 2026-10-25 carve-out that lets a dead generation's process key be
|
|
2343
|
-
// re-registered does not apply here: the group is alive, so the key is still held.
|
|
2344
|
-
await waitFor(() => statusRelease(daemon, "v1").state === "stopped")
|
|
2345
|
-
await retirementRecordSettled(daemon, "v1", "worker")
|
|
2346
|
-
|
|
2347
|
-
const worker = statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")
|
|
2348
|
-
|
|
2349
|
-
if (!worker?.heldProcessGroup) {
|
|
2350
|
-
throw new Error(`the drained generation should still hold the worker's process group, got ${JSON.stringify(worker && {pid: worker.pid, retirementTimedOut: worker.retirementTimedOut, state: worker.state})}`)
|
|
2351
|
-
}
|
|
2352
|
-
|
|
2353
|
-
await expect(daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})).rejects.toThrow(/Deploy refused before activation: release "v1" is retained in state stopped while 1 process of that generation still hold live work \(worker: process group \d+ with \d+ live member\(s\) still alive after the tracked process exited\)\. Activating that release id again would re-register those processes under a new provenance and abort with a guardian provenance mismatch\. The guardian's active release is "v2"/)
|
|
2354
|
-
expect(daemon.status().activeReleaseId).toBe("v2")
|
|
2355
|
-
expect(daemon.status().releases.filter((release) => release.releaseId === "v1").length).toBe(1)
|
|
2356
|
-
expect(await fetchText(daemon, "/release")).toBe("v2")
|
|
2357
|
-
} finally {
|
|
2358
|
-
await daemon.shutdown()
|
|
2359
|
-
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
2360
|
-
}
|
|
2361
|
-
})
|
|
2362
|
-
|
|
2363
1766
|
test("the rollback control command switches traffic over the socket", async () => {
|
|
2364
1767
|
const fixture = await createFixture()
|
|
2365
1768
|
const daemon = await startDaemon(fixture.config)
|
|
@@ -2567,8 +1970,8 @@ test("deploy can ensure the daemon before sending the release command", async ()
|
|
|
2567
1970
|
})
|
|
2568
1971
|
|
|
2569
1972
|
/**
|
|
2570
|
-
* @param {{companionReplicas?: number,
|
|
2571
|
-
* @returns {Promise<{activationGatePath: string, config: import("../src/config.js").RollbridgeConfig,
|
|
1973
|
+
* @param {{companionReplicas?: number, handoffService?: boolean, handoffServiceActivate?: boolean, handoffServiceActivateAmbiguousFailure?: boolean, handoffServiceActivateFailure?: boolean | string, handoffServiceQuiet?: boolean, handoffServiceQuietFailure?: boolean, includeCompanion?: boolean, includeService?: boolean, includeSingleton?: boolean, memoryLimitBytes?: number, nonBlockingDrainWorker?: boolean, persistState?: boolean, proxyHost?: string, singletonCwd?: string, webCommand?: string, webDependsOnService?: boolean, webHealthTimeoutMs?: number, workerReactivationFailure?: boolean, workerReactivationLifecycle?: boolean, workerStopDelayMs?: number}} [options] - Fixture options.
|
|
1974
|
+
* @returns {Promise<{activationGatePath: string, config: import("../src/config.js").RollbridgeConfig, lifecycleLogPath: string, retirementGatePath: string, root: string, serviceLogPath: string, serviceQuietPath: string, singletonLogPath: string, statePath: string}>} Fixture data.
|
|
2572
1975
|
*/
|
|
2573
1976
|
async function createFixture(options = {}) {
|
|
2574
1977
|
const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-test-"))
|
|
@@ -2579,7 +1982,6 @@ async function createFixture(options = {}) {
|
|
|
2579
1982
|
const retirementGatePath = path.join(root, "retirement.allow")
|
|
2580
1983
|
const singletonLogPath = path.join(root, "singleton.log")
|
|
2581
1984
|
const statePath = path.join(root, "rollbridge.state.json")
|
|
2582
|
-
const gatePath = path.join(root, "stc788-gate")
|
|
2583
1985
|
/** @type {Array<Record<string, import("../src/json.js").JsonValue>>} */
|
|
2584
1986
|
const processes = []
|
|
2585
1987
|
|
|
@@ -2649,11 +2051,9 @@ async function createFixture(options = {}) {
|
|
|
2649
2051
|
}
|
|
2650
2052
|
|
|
2651
2053
|
processes.push({
|
|
2652
|
-
command: options.webCommand || (options.
|
|
2653
|
-
? `${JSON.stringify(process.execPath)} ${JSON.stringify(
|
|
2654
|
-
:
|
|
2655
|
-
? `${JSON.stringify(process.execPath)} ${JSON.stringify(dependentAppPath)}`
|
|
2656
|
-
: `${JSON.stringify(process.execPath)} ${JSON.stringify(dummyAppPath)}`),
|
|
2054
|
+
command: options.webCommand || (options.webDependsOnService
|
|
2055
|
+
? `${JSON.stringify(process.execPath)} ${JSON.stringify(dependentAppPath)}`
|
|
2056
|
+
: `${JSON.stringify(process.execPath)} ${JSON.stringify(dummyAppPath)}`),
|
|
2657
2057
|
health: {
|
|
2658
2058
|
intervalMs: 50,
|
|
2659
2059
|
path: "/ping",
|
|
@@ -2693,51 +2093,7 @@ async function createFixture(options = {}) {
|
|
|
2693
2093
|
...((options.persistState || options.handoffServiceActivate) ? {ownerRecovery: {reconnectGraceMs: 30000}, statePath} : {})
|
|
2694
2094
|
})
|
|
2695
2095
|
|
|
2696
|
-
return {activationGatePath, config,
|
|
2697
|
-
}
|
|
2698
|
-
|
|
2699
|
-
/**
|
|
2700
|
-
* Config where both processes run the deferred-boot fixture, so a measured boot budget can be
|
|
2701
|
-
* compared against a dependency wait that must not be charged to it. Omitting
|
|
2702
|
-
* webHealthTimeoutMs leaves both the per-process and the release health budget unconfigured, so
|
|
2703
|
-
* the normalized config carries the default boot budget instead of a test-chosen one.
|
|
2704
|
-
* @param {{config: import("../src/config.js").RollbridgeConfig, root: string}} fixture - Fixture.
|
|
2705
|
-
* @param {{dependencyTimeoutMs: number, serviceBootMs: number, webBootMs: number, webHealthTimeoutMs?: number}} options - Budgets.
|
|
2706
|
-
* @returns {import("../src/config.js").RollbridgeConfig} Config for the deferred-boot processes.
|
|
2707
|
-
*/
|
|
2708
|
-
function deferredBootConfig(fixture, options) {
|
|
2709
|
-
const appCommand = `${JSON.stringify(process.execPath)} ${JSON.stringify(deferredBootAppPath)}`
|
|
2710
|
-
/** @type {Partial<import("../src/config.js").ProxyConfig>} */
|
|
2711
|
-
const proxyWithoutHealthBudget = structuredClone(fixture.config.proxy)
|
|
2712
|
-
|
|
2713
|
-
delete proxyWithoutHealthBudget.healthTimeoutMs
|
|
2714
|
-
|
|
2715
|
-
return normalizeConfig({
|
|
2716
|
-
...fixture.config,
|
|
2717
|
-
...(options.webHealthTimeoutMs === undefined ? {proxy: proxyWithoutHealthBudget} : {}),
|
|
2718
|
-
processes: [
|
|
2719
|
-
{
|
|
2720
|
-
command: appCommand,
|
|
2721
|
-
deployStrategy: "handoff",
|
|
2722
|
-
env: {ROLLBRIDGE_BOOT_MS: String(options.serviceBootMs)},
|
|
2723
|
-
id: "beacon",
|
|
2724
|
-
policy: "service",
|
|
2725
|
-
port: {from: 15000, to: 15099}
|
|
2726
|
-
},
|
|
2727
|
-
{
|
|
2728
|
-
command: appCommand,
|
|
2729
|
-
dependsOn: ["beacon"],
|
|
2730
|
-
dependencyTimeoutMs: options.dependencyTimeoutMs,
|
|
2731
|
-
env: {ROLLBRIDGE_BOOT_MS: String(options.webBootMs), ROLLBRIDGE_DEPENDENCY_PORT: "{{ports.beacon}}"},
|
|
2732
|
-
...(options.webHealthTimeoutMs === undefined
|
|
2733
|
-
? {}
|
|
2734
|
-
: {health: {intervalMs: 50, path: "/ping", timeoutMs: options.webHealthTimeoutMs}}),
|
|
2735
|
-
id: "web",
|
|
2736
|
-
policy: "proxied",
|
|
2737
|
-
port: {from: 0, to: 0}
|
|
2738
|
-
}
|
|
2739
|
-
]
|
|
2740
|
-
})
|
|
2096
|
+
return {activationGatePath, config, lifecycleLogPath, retirementGatePath, root, serviceLogPath, serviceQuietPath, singletonLogPath, statePath}
|
|
2741
2097
|
}
|
|
2742
2098
|
|
|
2743
2099
|
/**
|
|
@@ -2811,27 +2167,6 @@ function statusRelease(daemon, releaseId) {
|
|
|
2811
2167
|
return release
|
|
2812
2168
|
}
|
|
2813
2169
|
|
|
2814
|
-
/**
|
|
2815
|
-
* Waits until the daemon has observed the exit of a retiring process's tracked wrapper.
|
|
2816
|
-
*
|
|
2817
|
-
* A bounded retirement stop returns the moment its budget expires — that is the whole point of
|
|
2818
|
-
* the bound — while the wrapper it just signalled is still on its way out, so for a few
|
|
2819
|
-
* milliseconds the record still carries the wrapper pid and reports the process `quiesced`. The
|
|
2820
|
-
* residue the drain specs assert (the process group that outlived the tracked process) is
|
|
2821
|
-
* recorded by that exit observation, not by the stop call, so a spec that reads the record
|
|
2822
|
-
* straight after the stop returns races the observation it is describing. Measured on
|
|
2823
|
-
* 2026-11-16: `stopRelease` returned after 106 ms with `{pid: 13640, state: "quiesced"}` and the
|
|
2824
|
-
* record settled to `{heldProcessGroup: {liveMemberCount: 1, pgid: 13640}, state: "stopped"}`
|
|
2825
|
-
* within the next 50 ms.
|
|
2826
|
-
* @param {RollbridgeDaemon} daemon - Running daemon.
|
|
2827
|
-
* @param {string} releaseId - Retained generation to observe.
|
|
2828
|
-
* @param {string} processId - Process whose wrapper exit must be observed.
|
|
2829
|
-
* @returns {Promise<void>} Resolves once the process record has settled.
|
|
2830
|
-
*/
|
|
2831
|
-
async function retirementRecordSettled(daemon, releaseId, processId) {
|
|
2832
|
-
await waitFor(() => statusRelease(daemon, releaseId).processes.find((processStatus) => processStatus.id === processId)?.state === "stopped")
|
|
2833
|
-
}
|
|
2834
|
-
|
|
2835
2170
|
/**
|
|
2836
2171
|
* Maps process id to pid across the active release, services, and singletons.
|
|
2837
2172
|
* @param {import("../src/daemon.js").DaemonStatus} status - Daemon status payload.
|