rollbridge 0.1.72 → 0.1.74
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +63 -10
- package/TODO.md +13 -0
- package/changelog.d/20261025-guardian-re-register-dead-generation.md +13 -0
- package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +1 -0
- package/changelog.d/20261104-compose-service-units.md +21 -0
- package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +20 -0
- package/changelog.d/20261106-release-dependency-boot-budgets.md +32 -0
- package/changelog.d/20261109-path-filtered-builds.md +11 -0
- package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +26 -0
- package/changelog.d/20261111-contention-sensitive-daemon-specs.md +27 -0
- package/changelog.d/20261113-bounded-nonblocking-retirement.md +5 -0
- package/changelog.d/20261115-bound-draining-release-groups.md +62 -0
- package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +27 -0
- package/changelog.d/20261115-wedged-generation-escalation.md +44 -0
- package/changelog.d/20261116-bounded-explicit-stop-retirement.md +30 -0
- package/changelog.d/20261116-config-json-canonical-shape.md +11 -0
- package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +29 -0
- package/changelog.d/20261116-guardian-version-handoff.md +37 -0
- package/changelog.d/20261116-reclaim-dead-owner-mutation.md +7 -0
- package/changelog.d/20261117-activate-before-incumbent-retirement.md +1 -0
- package/changelog.d/20261117-activation-path-drain-deadline.md +35 -0
- package/changelog.d/20261117-stop-companion-after-retired-dependency.md +7 -0
- package/changelog.d/20261118-incumbent-retirement-authority-guard.md +1 -0
- package/changelog.d/20261118-owner-retirement-no-guardian-bound.md +19 -0
- package/changelog.d/20261118-shutdown-retirement-bound-restored.md +31 -0
- package/docs/cli.md +128 -3
- package/docs/compose-services.md +222 -0
- package/docs/config.md +177 -6
- package/docs/troubleshooting.md +97 -6
- package/docs/velocious.md +1 -1
- package/docs/workers.md +11 -0
- package/examples/tensorbuzz.com.js +3 -2
- package/package.json +1 -1
- package/src/cli.js +68 -2
- package/src/compose-unit.js +725 -0
- package/src/config.js +346 -8
- package/src/daemon.js +410 -73
- package/src/doctor.js +74 -0
- package/src/drain-policy.js +252 -0
- package/src/guardian-client.js +67 -8
- package/src/health.js +99 -8
- package/src/managed-process.js +197 -16
- package/src/process-guardian.js +212 -22
- package/src/process-memory.js +62 -2
- package/src/process-order.js +26 -0
- package/src/release-group.js +435 -37
- package/tensorbuzz.yml +20 -1
- package/test/completion.test.js +5 -2
- package/test/compose-config.test.js +160 -0
- package/test/compose-docker.test.js +297 -0
- package/test/compose-unit.test.js +415 -0
- package/test/config-validation.test.js +135 -5
- package/test/drain-policy.test.js +328 -0
- package/test/fixtures/deferred-boot-app.js +106 -0
- package/test/fixtures/gated-app.js +50 -0
- package/test/guardian-client.test.js +119 -5
- package/test/health.test.js +74 -1
- package/test/managed-process.test.js +260 -1
- package/test/owner-recovery.test.js +125 -21
- package/test/owner-replacement.test.js +175 -14
- package/test/process-memory.test.js +47 -1
- package/test/release-group.test.js +408 -2
- package/test/rollbridge.test.js +774 -72
- package/test/shutdown-completion.test.js +119 -0
- package/test/shutdown-retirement-bound.test.js +162 -0
- package/test/support/process.js +3 -1
- package/test/tensorbuzz-config.test.js +228 -0
package/test/health.test.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
3
|
import http from "node:http"
|
|
4
|
+
import net from "node:net"
|
|
4
5
|
import {describe, expect, test} from "@velocious/testing"
|
|
5
|
-
import {waitForHealth} from "../src/health.js"
|
|
6
|
+
import {waitForHealth, waitForListeningPort} from "../src/health.js"
|
|
6
7
|
|
|
7
8
|
describe("health", () => {
|
|
8
9
|
|
|
@@ -31,6 +32,18 @@ async function startHealthServer() {
|
|
|
31
32
|
}
|
|
32
33
|
}
|
|
33
34
|
|
|
35
|
+
/**
|
|
36
|
+
* @param {net.Server} server - Server to bind.
|
|
37
|
+
* @returns {Promise<number>} Ephemeral port it bound.
|
|
38
|
+
*/
|
|
39
|
+
async function bindPort(server) {
|
|
40
|
+
await new Promise((resolve) => server.listen(0, "127.0.0.1", () => resolve(undefined)))
|
|
41
|
+
|
|
42
|
+
const address = server.address()
|
|
43
|
+
|
|
44
|
+
return address && typeof address === "object" ? address.port : 0
|
|
45
|
+
}
|
|
46
|
+
|
|
34
47
|
test("waitForHealth delays the first probe by startDelayMs", async () => {
|
|
35
48
|
const server = await startHealthServer()
|
|
36
49
|
|
|
@@ -62,4 +75,64 @@ test("waitForHealth probes immediately when startDelayMs is 0", async () => {
|
|
|
62
75
|
await server.close()
|
|
63
76
|
}
|
|
64
77
|
})
|
|
78
|
+
|
|
79
|
+
test("waitForHealth names the deadline and the time it spent when the app never becomes healthy", async () => {
|
|
80
|
+
const server = http.createServer((_request, response) => {
|
|
81
|
+
response.writeHead(500)
|
|
82
|
+
response.end("bad")
|
|
83
|
+
})
|
|
84
|
+
|
|
85
|
+
try {
|
|
86
|
+
const port = await bindPort(server)
|
|
87
|
+
|
|
88
|
+
/** @type {import("../src/health.js").HealthCheckError} */
|
|
89
|
+
const failure = /** @type {import("../src/health.js").HealthCheckError} */ (await waitForHealth({
|
|
90
|
+
health: {intervalMs: 25, path: "/ping", startDelayMs: 0, timeoutMs: 300},
|
|
91
|
+
host: "127.0.0.1",
|
|
92
|
+
port,
|
|
93
|
+
processId: "web"
|
|
94
|
+
}).catch((error) => error instanceof Error ? error : new Error(String(error))))
|
|
95
|
+
|
|
96
|
+
expect(String(failure.message)).toMatch(/process web: \d+ probes, \d+ms elapsed of the 300ms health deadline/)
|
|
97
|
+
expect(failure.healthCheck?.deadlineMs).toBe(300)
|
|
98
|
+
expect({value: Boolean(failure.healthCheck && failure.healthCheck.elapsedMs >= 300), context: `expected the health gate to have spent its whole budget, was ${failure.healthCheck?.elapsedMs}ms`}).toMatchObject({value: true})
|
|
99
|
+
expect({value: Boolean(failure.healthCheck && failure.healthCheck.attempts >= 1), context: `expected at least one probe, was ${failure.healthCheck?.attempts}`}).toMatchObject({value: true})
|
|
100
|
+
expect(failure.healthCheck?.processId).toBe("web")
|
|
101
|
+
} finally {
|
|
102
|
+
await server.close()
|
|
103
|
+
}
|
|
104
|
+
})
|
|
105
|
+
|
|
106
|
+
test("waitForListeningPort waits for a dependency port on a budget of its own", async () => {
|
|
107
|
+
const server = net.createServer()
|
|
108
|
+
const port = await bindPort(server)
|
|
109
|
+
|
|
110
|
+
await new Promise((resolve) => server.close(() => resolve(undefined)))
|
|
111
|
+
|
|
112
|
+
const waiting = waitForListeningPort({host: "127.0.0.1", intervalMs: 25, port, timeoutMs: 2000})
|
|
113
|
+
|
|
114
|
+
await new Promise((resolve) => setTimeout(resolve, 300))
|
|
115
|
+
await new Promise((resolve) => server.listen(port, "127.0.0.1", () => resolve(undefined)))
|
|
116
|
+
|
|
117
|
+
try {
|
|
118
|
+
const result = await waiting
|
|
119
|
+
|
|
120
|
+
expect({value: Boolean(result.attempts >= 2), context: `expected more than one connection attempt, was ${result.attempts}`}).toMatchObject({value: true})
|
|
121
|
+
expect({value: Boolean(result.elapsedMs >= 250), context: `expected the dependency wait to take ~300ms, was ${result.elapsedMs}ms`}).toMatchObject({value: true})
|
|
122
|
+
} finally {
|
|
123
|
+
await new Promise((resolve) => server.close(() => resolve(undefined)))
|
|
124
|
+
}
|
|
125
|
+
})
|
|
126
|
+
|
|
127
|
+
test("waitForListeningPort names the budget it ran out on when the dependency never binds", async () => {
|
|
128
|
+
const server = net.createServer()
|
|
129
|
+
const port = await bindPort(server)
|
|
130
|
+
|
|
131
|
+
await new Promise((resolve) => server.close(() => resolve(undefined)))
|
|
132
|
+
|
|
133
|
+
/** @type {Error} */
|
|
134
|
+
const failure = /** @type {Error} */ (await waitForListeningPort({host: "127.0.0.1", intervalMs: 25, port, timeoutMs: 200}).catch((error) => error instanceof Error ? error : new Error(String(error))))
|
|
135
|
+
|
|
136
|
+
expect(String(failure.message)).toMatch(/did not accept a connection within 200ms/)
|
|
137
|
+
})
|
|
65
138
|
})
|
|
@@ -5,7 +5,7 @@ import os from "node:os"
|
|
|
5
5
|
import path from "node:path"
|
|
6
6
|
import {describe, expect, test} from "@velocious/testing"
|
|
7
7
|
import {fileURLToPath} from "node:url"
|
|
8
|
-
import ManagedProcess from "../src/managed-process.js"
|
|
8
|
+
import ManagedProcess, {childRecordIsGone} from "../src/managed-process.js"
|
|
9
9
|
|
|
10
10
|
describe("managed-process", () => {
|
|
11
11
|
|
|
@@ -308,6 +308,255 @@ test("runs quiet and drain lifecycle hooks before stopping", async () => {
|
|
|
308
308
|
}
|
|
309
309
|
})
|
|
310
310
|
|
|
311
|
+
test("retirement timeout leaves an indefinite worker alive without SIGKILL", async () => {
|
|
312
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-retirement-wedge-"))
|
|
313
|
+
const readyPath = path.join(dir, "ready")
|
|
314
|
+
/** @type {string[]} */
|
|
315
|
+
const signals = []
|
|
316
|
+
const managed = new ManagedProcess({
|
|
317
|
+
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`process.on('SIGTERM', () => {}); require("node:fs").writeFileSync(${JSON.stringify(readyPath)}, "ready"); setInterval(() => {}, 1000)`)}`,
|
|
318
|
+
cwd: undefined,
|
|
319
|
+
env: {},
|
|
320
|
+
id: "worker",
|
|
321
|
+
lifecycle: {drainTimeoutMs: 0},
|
|
322
|
+
logger: () => {},
|
|
323
|
+
outputLines: 50,
|
|
324
|
+
restartDelayMs: 10,
|
|
325
|
+
shouldRestart: () => false,
|
|
326
|
+
stopSignal: "SIGTERM",
|
|
327
|
+
stopTimeoutMs: "indefinite"
|
|
328
|
+
})
|
|
329
|
+
const signalProcessGroup = managed.signalProcessGroup.bind(managed)
|
|
330
|
+
managed.signalProcessGroup = async (signal, pgid, deadline) => {
|
|
331
|
+
signals.push(signal)
|
|
332
|
+
await signalProcessGroup(signal, pgid, deadline)
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
try {
|
|
336
|
+
await managed.start()
|
|
337
|
+
// The fixture installs its SIGTERM disposition before it announces readiness. Signalling a
|
|
338
|
+
// child that has not finished booting reaches the default disposition instead, which exits
|
|
339
|
+
// the group and leaves no wedge to observe.
|
|
340
|
+
await waitFor(() => fs.existsSync(readyPath))
|
|
341
|
+
await managed.stop({retirementTimeoutMs: 25})
|
|
342
|
+
|
|
343
|
+
expect(managed.retirementTimedOut).toBe(true)
|
|
344
|
+
expect(signals).toEqual(["SIGTERM"])
|
|
345
|
+
expect(managed.status().pid).toBeDefined()
|
|
346
|
+
} finally {
|
|
347
|
+
const pid = managed.status().pid
|
|
348
|
+
if (pid) process.kill(-pid, "SIGKILL")
|
|
349
|
+
if (managed.exitPromise) await managed.exitPromise
|
|
350
|
+
fs.rmSync(dir, {force: true, recursive: true})
|
|
351
|
+
}
|
|
352
|
+
})
|
|
353
|
+
|
|
354
|
+
test("escalateRetirement terminates the wedged worker the bounded retirement left alive", async () => {
|
|
355
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-escalation-wedge-"))
|
|
356
|
+
const readyPath = path.join(dir, "ready")
|
|
357
|
+
/** @type {string[]} */
|
|
358
|
+
const signals = []
|
|
359
|
+
/** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
|
|
360
|
+
const logs = []
|
|
361
|
+
const managed = new ManagedProcess({
|
|
362
|
+
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`process.on('SIGTERM', () => {}); require("node:fs").writeFileSync(${JSON.stringify(readyPath)}, "ready"); setInterval(() => {}, 1000)`)}`,
|
|
363
|
+
cwd: undefined,
|
|
364
|
+
env: {},
|
|
365
|
+
id: "worker",
|
|
366
|
+
lifecycle: {drainTimeoutMs: 0},
|
|
367
|
+
logger: (message, data) => logs.push({data, message}),
|
|
368
|
+
outputLines: 50,
|
|
369
|
+
restartDelayMs: 10,
|
|
370
|
+
shouldRestart: () => false,
|
|
371
|
+
stopSignal: "SIGTERM",
|
|
372
|
+
stopTimeoutMs: "indefinite"
|
|
373
|
+
})
|
|
374
|
+
const signalProcessGroup = managed.signalProcessGroup.bind(managed)
|
|
375
|
+
|
|
376
|
+
managed.signalProcessGroup = async (signal, pgid, deadline) => {
|
|
377
|
+
signals.push(signal)
|
|
378
|
+
await signalProcessGroup(signal, pgid, deadline)
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
try {
|
|
382
|
+
await managed.start()
|
|
383
|
+
await waitFor(() => fs.existsSync(readyPath))
|
|
384
|
+
await managed.stop({retirementTimeoutMs: 25})
|
|
385
|
+
const wedgedPid = managed.status().pid
|
|
386
|
+
|
|
387
|
+
expect(managed.retirementTimedOut).toBe(true)
|
|
388
|
+
expect(signals).toEqual(["SIGTERM"])
|
|
389
|
+
expect(wedgedPid).toBeDefined()
|
|
390
|
+
|
|
391
|
+
expect(await managed.escalateRetirement({reason: "spec: proven wedge"})).toBe(true)
|
|
392
|
+
expect(signals).toEqual(["SIGTERM", "SIGKILL"])
|
|
393
|
+
expect(logs.some((entry) => entry.message === "process retirement escalated to SIGKILL" && entry.data?.reason === "spec: proven wedge")).toBe(true)
|
|
394
|
+
expect(await managed.waitForProcessGroupExit(/** @type {number} */ (wedgedPid), Date.now() + 5000)).toBe(true)
|
|
395
|
+
if (managed.exitPromise) await managed.exitPromise
|
|
396
|
+
expect(managed.status().pid).toBeUndefined()
|
|
397
|
+
expect(managed.status().state).toBe("stopped")
|
|
398
|
+
} finally {
|
|
399
|
+
const pid = managed.status().pid
|
|
400
|
+
if (pid) process.kill(-pid, "SIGKILL")
|
|
401
|
+
if (managed.exitPromise) await managed.exitPromise
|
|
402
|
+
fs.rmSync(dir, {force: true, recursive: true})
|
|
403
|
+
}
|
|
404
|
+
})
|
|
405
|
+
|
|
406
|
+
test("escalateRetirement refuses a process that has not expired its retirement budget", async () => {
|
|
407
|
+
/** @type {string[]} */
|
|
408
|
+
const signals = []
|
|
409
|
+
/** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
|
|
410
|
+
const logs = []
|
|
411
|
+
const managed = new ManagedProcess({
|
|
412
|
+
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify("setInterval(() => {}, 1000)")}`,
|
|
413
|
+
cwd: undefined,
|
|
414
|
+
env: {},
|
|
415
|
+
id: "worker",
|
|
416
|
+
lifecycle: {drainTimeoutMs: 0},
|
|
417
|
+
logger: (message, data) => logs.push({data, message}),
|
|
418
|
+
outputLines: 50,
|
|
419
|
+
restartDelayMs: 10,
|
|
420
|
+
shouldRestart: () => false,
|
|
421
|
+
stopSignal: "SIGTERM",
|
|
422
|
+
stopTimeoutMs: "indefinite"
|
|
423
|
+
})
|
|
424
|
+
const signalProcessGroup = managed.signalProcessGroup.bind(managed)
|
|
425
|
+
|
|
426
|
+
managed.signalProcessGroup = async (signal, pgid, deadline) => {
|
|
427
|
+
signals.push(signal)
|
|
428
|
+
await signalProcessGroup(signal, pgid, deadline)
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
try {
|
|
432
|
+
await managed.start()
|
|
433
|
+
await managed.quiesce()
|
|
434
|
+
|
|
435
|
+
expect(managed.retirementTimedOut).toBe(false)
|
|
436
|
+
expect(await managed.escalateRetirement({reason: "spec"})).toBe(false)
|
|
437
|
+
expect(signals).toEqual([])
|
|
438
|
+
expect(logs.some((entry) => entry.message === "retirement escalation refused; process is not past its retirement budget")).toBe(true)
|
|
439
|
+
expect(managed.status().pid).toBeDefined()
|
|
440
|
+
} finally {
|
|
441
|
+
const pid = managed.status().pid
|
|
442
|
+
if (pid) process.kill(-pid, "SIGKILL")
|
|
443
|
+
if (managed.exitPromise) await managed.exitPromise
|
|
444
|
+
}
|
|
445
|
+
})
|
|
446
|
+
|
|
447
|
+
test("escalateRetirement is inert for a process that drained normally", async () => {
|
|
448
|
+
/** @type {string[]} */
|
|
449
|
+
const signals = []
|
|
450
|
+
const managed = new ManagedProcess({
|
|
451
|
+
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify("setInterval(() => {}, 1000)")}`,
|
|
452
|
+
cwd: undefined,
|
|
453
|
+
env: {},
|
|
454
|
+
id: "worker",
|
|
455
|
+
lifecycle: {drainTimeoutMs: 0},
|
|
456
|
+
logger: () => {},
|
|
457
|
+
outputLines: 50,
|
|
458
|
+
restartDelayMs: 10,
|
|
459
|
+
shouldRestart: () => false,
|
|
460
|
+
stopSignal: "SIGTERM",
|
|
461
|
+
stopTimeoutMs: 2000
|
|
462
|
+
})
|
|
463
|
+
const signalProcessGroup = managed.signalProcessGroup.bind(managed)
|
|
464
|
+
|
|
465
|
+
managed.signalProcessGroup = async (signal, pgid, deadline) => {
|
|
466
|
+
signals.push(signal)
|
|
467
|
+
await signalProcessGroup(signal, pgid, deadline)
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
try {
|
|
471
|
+
await managed.start()
|
|
472
|
+
await managed.stop()
|
|
473
|
+
|
|
474
|
+
expect(managed.status().state).toBe("stopped")
|
|
475
|
+
expect(await managed.escalateRetirement({reason: "spec"})).toBe(false)
|
|
476
|
+
expect(signals).toEqual(["SIGTERM"])
|
|
477
|
+
} finally {
|
|
478
|
+
const pid = managed.status().pid
|
|
479
|
+
if (pid) process.kill(-pid, "SIGKILL")
|
|
480
|
+
if (managed.exitPromise) await managed.exitPromise
|
|
481
|
+
}
|
|
482
|
+
})
|
|
483
|
+
|
|
484
|
+
test("a retirement budget caps a finite graceful window instead of being outvoted by it", async () => {
|
|
485
|
+
/** @type {string[]} */
|
|
486
|
+
const signals = []
|
|
487
|
+
/** @type {Array<{msFromStopStart: number, phase: string}>} */
|
|
488
|
+
const deadlines = []
|
|
489
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-retirement-bound-"))
|
|
490
|
+
const readyPath = path.join(dir, "ready")
|
|
491
|
+
const managed = new ManagedProcess({
|
|
492
|
+
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`process.on('SIGTERM', () => {}); require("node:fs").writeFileSync(${JSON.stringify(readyPath)}, "ready"); setInterval(() => {}, 1000)`)}`,
|
|
493
|
+
cwd: undefined,
|
|
494
|
+
env: {},
|
|
495
|
+
id: "worker",
|
|
496
|
+
lifecycle: {drainTimeoutMs: 0},
|
|
497
|
+
logger: () => {},
|
|
498
|
+
outputLines: 50,
|
|
499
|
+
restartDelayMs: 10,
|
|
500
|
+
shouldRestart: () => false,
|
|
501
|
+
stopSignal: "SIGTERM",
|
|
502
|
+
stopTimeoutMs: 2000
|
|
503
|
+
})
|
|
504
|
+
let stopStartedAtMs = 0
|
|
505
|
+
|
|
506
|
+
/**
|
|
507
|
+
* @param {string} phase - The step the deadline was handed to.
|
|
508
|
+
* @param {number | undefined} deadline - Absolute deadline the step received.
|
|
509
|
+
* @returns {void} Records the deadline as an offset from the start of the stop.
|
|
510
|
+
*/
|
|
511
|
+
const recordDeadline = (phase, deadline) => {
|
|
512
|
+
if (deadline !== undefined) deadlines.push({msFromStopStart: deadline - stopStartedAtMs, phase})
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
const signalProcessGroup = managed.signalProcessGroup.bind(managed)
|
|
516
|
+
managed.signalProcessGroup = async (signal, pgid, deadline) => {
|
|
517
|
+
signals.push(signal)
|
|
518
|
+
recordDeadline("signal", deadline)
|
|
519
|
+
await signalProcessGroup(signal, pgid, deadline)
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
const waitForProcessGroupExit = managed.waitForProcessGroupExit.bind(managed)
|
|
523
|
+
managed.waitForProcessGroupExit = async (pgid, deadline) => {
|
|
524
|
+
recordDeadline("wait", deadline)
|
|
525
|
+
return await waitForProcessGroupExit(pgid, deadline)
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
try {
|
|
529
|
+
await managed.start()
|
|
530
|
+
// The fixture installs its SIGTERM disposition before it announces readiness. Signalling a
|
|
531
|
+
// child that has not finished booting reaches the default disposition instead, which exits
|
|
532
|
+
// the group and leaves no retirement wait to observe.
|
|
533
|
+
await waitFor(() => fs.existsSync(readyPath))
|
|
534
|
+
stopStartedAtMs = Date.now()
|
|
535
|
+
|
|
536
|
+
await managed.stop({retirementTimeoutMs: 25})
|
|
537
|
+
|
|
538
|
+
// The 2 000 ms graceful window must not outvote the 25 ms retirement budget. Read the other
|
|
539
|
+
// way round the wait runs the whole window before it escalates to SIGKILL, which is the shape
|
|
540
|
+
// that held the TensorBuzz control plane for 4,5 h after the worker moved from "indefinite" to
|
|
541
|
+
// 4,5 h and killed the in-flight builds the consumer said not to interrupt.
|
|
542
|
+
expect(deadlines.map((entry) => entry.phase)).toEqual(["signal", "wait"])
|
|
543
|
+
for (const entry of deadlines) {
|
|
544
|
+
// The deadline starts at stop entry; a millisecond clock tick between this test's marker
|
|
545
|
+
// and the async stop call may shave 1 ms from the observed offset.
|
|
546
|
+
expect(entry.msFromStopStart).toBeGreaterThanOrEqual(0)
|
|
547
|
+
expect(entry.msFromStopStart).toBeLessThan(1000)
|
|
548
|
+
}
|
|
549
|
+
expect(managed.retirementTimedOut).toBe(true)
|
|
550
|
+
expect(signals).toEqual(["SIGTERM"])
|
|
551
|
+
expect(managed.status().pid).toBeDefined()
|
|
552
|
+
} finally {
|
|
553
|
+
const pid = managed.status().pid
|
|
554
|
+
if (pid) process.kill(-pid, "SIGKILL")
|
|
555
|
+
if (managed.exitPromise) await managed.exitPromise
|
|
556
|
+
fs.rmSync(dir, {force: true, recursive: true})
|
|
557
|
+
}
|
|
558
|
+
})
|
|
559
|
+
|
|
311
560
|
test("a configured stopCommand is used instead of the stop signal", async () => {
|
|
312
561
|
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-hooks-"))
|
|
313
562
|
const logPath = path.join(dir, "stop.log")
|
|
@@ -894,4 +1143,14 @@ test("the unlimited constant-delay fast path still applies maxDelayMs", () => {
|
|
|
894
1143
|
|
|
895
1144
|
expect(queued).toBe(5)
|
|
896
1145
|
})
|
|
1146
|
+
|
|
1147
|
+
test("the guardian fence releases only a record that is terminal and holds no process group", () => {
|
|
1148
|
+
expect(childRecordIsGone({heldProcessGroup: undefined, pid: undefined, state: "stopped"})).toBe(true)
|
|
1149
|
+
expect(childRecordIsGone({heldProcessGroup: undefined, pid: undefined, state: "failed"})).toBe(true)
|
|
1150
|
+
expect(childRecordIsGone({heldProcessGroup: undefined, pid: 4242, state: "running"})).toBe(false)
|
|
1151
|
+
|
|
1152
|
+
// The shape a drained generation leaves behind: the tracked process record is terminal, but the
|
|
1153
|
+
// group it led still holds live members, so the key is still held by that generation.
|
|
1154
|
+
expect(childRecordIsGone({heldProcessGroup: {liveMemberCount: 1, pgid: 4242}, pid: undefined, state: "stopped"})).toBe(false)
|
|
1155
|
+
})
|
|
897
1156
|
})
|
|
@@ -11,7 +11,7 @@ import {fileURLToPath} from "node:url"
|
|
|
11
11
|
import {normalizeConfig} from "../src/config.js"
|
|
12
12
|
import {sendControlCommand} from "../src/control-client.js"
|
|
13
13
|
import RollbridgeDaemon from "../src/daemon.js"
|
|
14
|
-
import GuardianClient from "../src/guardian-client.js"
|
|
14
|
+
import GuardianClient, {usableSocketPath} from "../src/guardian-client.js"
|
|
15
15
|
import {isProcessRunning, waitForProcessExit} from "./support/process.js"
|
|
16
16
|
|
|
17
17
|
describe("owner-recovery", () => {
|
|
@@ -26,7 +26,7 @@ const serviceAppPath = path.join(currentDir, "fixtures", "service-app.js")
|
|
|
26
26
|
|
|
27
27
|
test("external owner retirement releases guardian authority without losing its generation", async () => {
|
|
28
28
|
const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-owner-retirement-recovery-"))
|
|
29
|
-
const socketPath = path.join(root, "rollbridge.sock")
|
|
29
|
+
const socketPath = usableSocketPath(path.join(root, "rollbridge.sock"))
|
|
30
30
|
const statePath = path.join(root, "rollbridge.state.json")
|
|
31
31
|
const v1Path = path.join(root, "v1")
|
|
32
32
|
const v2Path = path.join(root, "v2")
|
|
@@ -710,13 +710,17 @@ test("guardian recovery becomes ready before replaying a gated generation hook",
|
|
|
710
710
|
|
|
711
711
|
expect(recovering.daemonPid).toBe(recoveredDaemonPid)
|
|
712
712
|
expect(recovering.ownerRecovery?.ready).toBe(true)
|
|
713
|
-
expect(recovering.generationTransition?.phase).toBe("
|
|
713
|
+
expect(recovering.generationTransition?.phase).toBe("committed_pending")
|
|
714
714
|
expect(typeof guardianPid).toBe("number")
|
|
715
|
+
await expect(() => sendControlCommand({command: {command: "stop", releaseId: "v1"}, path: fixture.socketPath})).toThrow(/Another owner mutation/)
|
|
716
|
+
await expect(() => sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})).toThrow(/Cannot shut down while generation transition recovery is in progress/)
|
|
717
|
+
await expect(() => sendControlCommand({command: {attestation: `sha256:${"a".repeat(64)}`, command: "retire-owner"}, path: fixture.socketPath})).toThrow(/Cannot retire owner while generation transition recovery is in progress/)
|
|
718
|
+
|
|
715
719
|
process.kill(recoveredDaemonPid, "SIGTERM")
|
|
716
720
|
await fs.writeFile(retirementGatePath, "release retirement\n")
|
|
717
721
|
await waitForProcessExit(recoveredDaemonPid, 5000)
|
|
718
722
|
await waitForProcessExit(guardianPid, 5000)
|
|
719
|
-
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2", "retire:
|
|
723
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2", "retire:v2", "retire:v1"])
|
|
720
724
|
} finally {
|
|
721
725
|
await fs.writeFile(retirementGatePath, "release retirement\n").catch(() => undefined)
|
|
722
726
|
await sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath}).catch(() => undefined)
|
|
@@ -858,7 +862,7 @@ test("owner recovery preserves complete private transition authority across a ca
|
|
|
858
862
|
}
|
|
859
863
|
})
|
|
860
864
|
|
|
861
|
-
test("
|
|
865
|
+
test("owner recovery reconstructs a stopped release that still owns a committed-pending singleton", async () => {
|
|
862
866
|
const fixture = await createFixture()
|
|
863
867
|
|
|
864
868
|
fixture.config.releaseRetention = {keep: 0, maxAgeMs: 0}
|
|
@@ -866,16 +870,23 @@ test("committed-pending generation retains the incumbent until singleton replace
|
|
|
866
870
|
const config = normalizeConfig(fixture.config, fixture.configPath)
|
|
867
871
|
const owner = new RollbridgeDaemon({config, configPath: fixture.configPath, logger: () => {}})
|
|
868
872
|
const replacementPause = pauseSingletonReplacement(owner, "v2")
|
|
873
|
+
const capturedDrain = captureReleaseDrain(owner, "v1")
|
|
869
874
|
const v1Path = await prepareRelease(fixture.root, "v1")
|
|
870
875
|
const v2Path = await prepareRelease(fixture.root, "v2")
|
|
871
876
|
/** @type {Promise<Record<string, import("../src/json.js").JsonValue>> | undefined} */
|
|
872
877
|
let deployPromise
|
|
878
|
+
/** @type {RollbridgeDaemon | undefined} */
|
|
879
|
+
let recovered
|
|
880
|
+
|
|
873
881
|
try {
|
|
874
882
|
await owner.start()
|
|
875
883
|
await owner.deploy({releaseId: "v1", releasePath: v1Path, revision: "v1"})
|
|
876
884
|
deployPromise = owner.deploy({releaseId: "v2", releasePath: v2Path, revision: "v2"})
|
|
877
885
|
void deployPromise.catch(() => {})
|
|
878
886
|
await replacementPause.started
|
|
887
|
+
await capturedDrain.started
|
|
888
|
+
await fs.writeFile(path.join(v1Path, "worker.fifo"), "drained\n")
|
|
889
|
+
await capturedDrain.completed
|
|
879
890
|
if (owner.pendingWrite) await owner.pendingWrite
|
|
880
891
|
const pending = /** @type {RecoveryState} */ (JSON.parse(await fs.readFile(fixture.statePath, "utf8")))
|
|
881
892
|
|
|
@@ -883,19 +894,38 @@ test("committed-pending generation retains the incumbent until singleton replace
|
|
|
883
894
|
expect(pending.generationTransition?.phase).toBe("committed_pending")
|
|
884
895
|
expect(pending.serviceReleaseIds?.beacon).toBe("v2")
|
|
885
896
|
expect(pending.singletonReleaseIds?.singleton).toBe("v1")
|
|
897
|
+
const stoppedRelease = pending.releases.find((release) => release.releaseId === "v1" && release.state === "stopped")
|
|
898
|
+
|
|
899
|
+
if (!stoppedRelease) throw new Error("Missing stopped singleton owner release")
|
|
886
900
|
expect(pending.releaseReferences.map((reference) => reference.releaseId)).toEqual(["v1", "v2"])
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
901
|
+
await owner.retireCommittedOwner(undefined)
|
|
902
|
+
owner.guardian?.disconnect()
|
|
903
|
+
|
|
904
|
+
recovered = new RollbridgeDaemon({config, configPath: fixture.configPath, logger: () => {}})
|
|
905
|
+
await recovered.start()
|
|
906
|
+
|
|
907
|
+
expect(recovered.status().generationTransition?.phase).toBe("committed")
|
|
908
|
+
expect(recovered.singletonReleaseIds.get("singleton")).toBe("v2")
|
|
909
|
+
// the stopped singleton owner may be pruned after replacement commits
|
|
910
|
+
expect(!recovered.releases.has("v1")).toBeTruthy()
|
|
911
|
+
expect(recovered.portReservations.has(stoppedRelease.ports.jobs)).toBe(false)
|
|
912
|
+
expect(recovered.portReservations.has(stoppedRelease.ports.web)).toBe(false)
|
|
892
913
|
} finally {
|
|
893
914
|
replacementPause.continue()
|
|
894
915
|
await deployPromise?.catch(() => {})
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
916
|
+
if (recovered) {
|
|
917
|
+
const shutdown = recovered.shutdown()
|
|
918
|
+
|
|
919
|
+
await fs.writeFile(path.join(v2Path, "worker.fifo"), "drained\n").catch(() => undefined)
|
|
920
|
+
await shutdown.catch(() => {})
|
|
921
|
+
recovered.guardian?.disconnect()
|
|
922
|
+
} else {
|
|
923
|
+
const shutdown = owner.shutdown()
|
|
924
|
+
|
|
925
|
+
await fs.writeFile(path.join(v2Path, "worker.fifo"), "drained\n").catch(() => undefined)
|
|
926
|
+
await shutdown.catch(() => {})
|
|
927
|
+
owner.guardian?.disconnect()
|
|
928
|
+
}
|
|
899
929
|
await stopFixtureGuardian(fixture.statePath)
|
|
900
930
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
901
931
|
}
|
|
@@ -973,7 +1003,7 @@ test("owner recovery uses the owning release singleton definition during a commi
|
|
|
973
1003
|
}
|
|
974
1004
|
})
|
|
975
1005
|
|
|
976
|
-
test("owner recovery retains
|
|
1006
|
+
test("owner recovery retains a stopped previous release until committed-pending singleton work completes", async () => {
|
|
977
1007
|
const fixture = await createFixture()
|
|
978
1008
|
const processes = /** @type {Record<string, import("../src/json.js").JsonValue>[]} */ (fixture.config.processes)
|
|
979
1009
|
const worker = processes.find((processConfig) => processConfig.id === "worker")
|
|
@@ -1011,12 +1041,10 @@ test("owner recovery retains the committed generation after singleton work compl
|
|
|
1011
1041
|
deployPromise = owner.deploy({releaseId: "v2", releasePath: v2Path, revision: "v2"})
|
|
1012
1042
|
void deployPromise.catch(() => {})
|
|
1013
1043
|
await replacementComplete
|
|
1014
|
-
const pending = await waitForState(fixture.statePath, (state) => state.generationTransition?.phase === "committed_pending", AbortSignal.timeout(5000))
|
|
1044
|
+
const pending = await waitForState(fixture.statePath, (state) => state.generationTransition?.phase === "committed_pending" && state.releases?.some((release) => release.releaseId === "v1" && release.state === "stopped"), AbortSignal.timeout(5000))
|
|
1015
1045
|
|
|
1016
1046
|
expect(pending.releaseReferences.map((reference) => reference.releaseId)).toEqual(["v1", "v2"])
|
|
1017
|
-
expect(pending.singletonReleaseIds?.singleton).toBe("
|
|
1018
|
-
continueReplacement()
|
|
1019
|
-
await deployPromise
|
|
1047
|
+
expect(pending.singletonReleaseIds?.singleton).toBe("v2")
|
|
1020
1048
|
await owner.retireCommittedOwner(undefined)
|
|
1021
1049
|
owner.guardian?.disconnect()
|
|
1022
1050
|
|
|
@@ -1070,8 +1098,11 @@ test("owner recovery replays ambiguous retirement with the previous release's ex
|
|
|
1070
1098
|
await fs.writeFile(retirementGatePath, "release retirement\n")
|
|
1071
1099
|
|
|
1072
1100
|
owner = spawnDaemon(fixture.configPath)
|
|
1101
|
+
const recoverySettled = waitForLog(owner, "release generation transition recovery settled")
|
|
1102
|
+
|
|
1073
1103
|
await waitForLog(owner, "control socket listening")
|
|
1074
1104
|
await waitForState(fixture.statePath, (state) => state.generationTransition?.phase === "committed", AbortSignal.timeout(5000))
|
|
1105
|
+
await recoverySettled
|
|
1075
1106
|
// ambiguous retirement must replay exactly once
|
|
1076
1107
|
expect((await fs.readFile(retirementWaitingPath, "utf8")).trim().split("\n").length).toBe(1)
|
|
1077
1108
|
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2", "retire:v1"])
|
|
@@ -1652,13 +1683,52 @@ test("ensure-daemon atomically replaces an incompatible owner without losing ret
|
|
|
1652
1683
|
}
|
|
1653
1684
|
})
|
|
1654
1685
|
|
|
1686
|
+
test("startup against a version-drifted guardian without an incumbent refuses and names both versions and the repair command", async () => {
|
|
1687
|
+
const fixture = await createFixture()
|
|
1688
|
+
const config = normalizeConfig(fixture.config, fixture.configPath)
|
|
1689
|
+
const owner = new RollbridgeDaemon({config, configPath: fixture.configPath, logger: () => {}})
|
|
1690
|
+
const v1Path = await prepareRelease(fixture.root, "v1")
|
|
1691
|
+
/** @type {RollbridgeDaemon | undefined} */
|
|
1692
|
+
let drifted
|
|
1693
|
+
|
|
1694
|
+
try {
|
|
1695
|
+
await owner.start()
|
|
1696
|
+
await owner.deploy({releaseId: "v1", releasePath: v1Path, revision: "v1"})
|
|
1697
|
+
|
|
1698
|
+
const workerPid = releaseProcessPid(owner.status(), "v1", "worker")
|
|
1699
|
+
const guardianVersion = JSON.parse(await fs.readFile(path.join(currentDir, "..", "package.json"), "utf8")).version
|
|
1700
|
+
drifted = new RollbridgeDaemon({
|
|
1701
|
+
config,
|
|
1702
|
+
configPath: fixture.configPath,
|
|
1703
|
+
logger: () => {},
|
|
1704
|
+
runtime: {digest: "f".repeat(64), format: 1, path: path.join(fixture.root, "drift-runtime"), version: "0.0.1"}
|
|
1705
|
+
})
|
|
1706
|
+
|
|
1707
|
+
const refusal = await drifted.start().then(() => undefined, (error) => error)
|
|
1708
|
+
const message = refusal instanceof Error ? refusal.message : ""
|
|
1709
|
+
|
|
1710
|
+
expect(message).toContain(`The persistent Rollbridge guardian is version ${guardianVersion} while this daemon runs rollbridge 0.0.1`)
|
|
1711
|
+
expect(message).toContain("this startup path has no incumbent daemon to hand guardian ownership from")
|
|
1712
|
+
expect(message).toContain(`Run "rollbridge shutdown --config ${fixture.configPath}" and then "rollbridge ensure-daemon --config ${fixture.configPath}"`)
|
|
1713
|
+
expect(isProcessRunning(workerPid)).toBe(true)
|
|
1714
|
+
} finally {
|
|
1715
|
+
drifted?.guardian?.disconnect()
|
|
1716
|
+
await Promise.all([
|
|
1717
|
+
fs.writeFile(path.join(v1Path, "worker.fifo"), "drained\n").catch(() => {}),
|
|
1718
|
+
owner.shutdown().catch(() => {})
|
|
1719
|
+
])
|
|
1720
|
+
await stopFixtureGuardian(fixture.statePath)
|
|
1721
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
1722
|
+
}
|
|
1723
|
+
})
|
|
1724
|
+
|
|
1655
1725
|
/**
|
|
1656
1726
|
* @param {{activationFailureRelease?: string}} [options] - Lifecycle fault injection.
|
|
1657
1727
|
* @returns {Promise<{activationGatePath: string, config: Record<string, import("../src/json.js").JsonValue>, configPath: string, lifecycleLogPath: string, root: string, socketPath: string, statePath: string}>} Fixture paths.
|
|
1658
1728
|
*/
|
|
1659
1729
|
async function createFixture(options = {}) {
|
|
1660
1730
|
const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-owner-recovery-"))
|
|
1661
|
-
const socketPath = path.join(root, "rollbridge.sock")
|
|
1731
|
+
const socketPath = usableSocketPath(path.join(root, "rollbridge.sock"))
|
|
1662
1732
|
const configPath = path.join(root, "rollbridge.cjs")
|
|
1663
1733
|
const statePath = path.join(root, "rollbridge.state.json")
|
|
1664
1734
|
const serviceLogPath = path.join(root, "service.log")
|
|
@@ -1806,6 +1876,40 @@ function pauseSingletonReplacement(daemon, releaseId) {
|
|
|
1806
1876
|
return {continue: continueReplacement, started}
|
|
1807
1877
|
}
|
|
1808
1878
|
|
|
1879
|
+
/**
|
|
1880
|
+
* Captures one background release drain through its package-owned promise.
|
|
1881
|
+
* @param {RollbridgeDaemon} daemon - Current owner.
|
|
1882
|
+
* @param {string} releaseId - Release whose drain completion is required.
|
|
1883
|
+
* @returns {{completed: Promise<void>, started: Promise<void>}} Drain boundaries.
|
|
1884
|
+
*/
|
|
1885
|
+
function captureReleaseDrain(daemon, releaseId) {
|
|
1886
|
+
const drainAndPrune = daemon.drainAndPrune.bind(daemon)
|
|
1887
|
+
/** @type {() => void} */
|
|
1888
|
+
let markStarted = () => {}
|
|
1889
|
+
/** @type {(error: Error | string) => void} */
|
|
1890
|
+
let rejectCompleted = () => {}
|
|
1891
|
+
/** @type {() => void} */
|
|
1892
|
+
let resolveCompleted = () => {}
|
|
1893
|
+
const started = new Promise((resolve) => { markStarted = () => resolve(undefined) })
|
|
1894
|
+
const completed = new Promise((resolve, reject) => {
|
|
1895
|
+
resolveCompleted = () => resolve(undefined)
|
|
1896
|
+
rejectCompleted = reject
|
|
1897
|
+
})
|
|
1898
|
+
|
|
1899
|
+
daemon.drainAndPrune = async (release, config) => {
|
|
1900
|
+
if (release.releaseId !== releaseId) return await drainAndPrune(release, config)
|
|
1901
|
+
markStarted()
|
|
1902
|
+
try {
|
|
1903
|
+
await drainAndPrune(release, config)
|
|
1904
|
+
resolveCompleted()
|
|
1905
|
+
} catch (error) {
|
|
1906
|
+
rejectCompleted(error instanceof Error ? error : String(error))
|
|
1907
|
+
throw error
|
|
1908
|
+
}
|
|
1909
|
+
}
|
|
1910
|
+
return {completed, started}
|
|
1911
|
+
}
|
|
1912
|
+
|
|
1809
1913
|
/**
|
|
1810
1914
|
* @param {string} statePath - Fixture state path.
|
|
1811
1915
|
* @returns {Promise<void>} Cleanup completion.
|
|
@@ -1818,7 +1922,7 @@ async function stopFixtureGuardian(statePath) {
|
|
|
1818
1922
|
if (!identity || typeof identity.pid !== "number" || typeof identity.socketPath !== "string" || typeof identity.token !== "string") return
|
|
1819
1923
|
const command = (await fs.readFile(`/proc/${identity.pid}/cmdline`)).toString().replaceAll("\0", " ")
|
|
1820
1924
|
|
|
1821
|
-
if (!command.includes("process-guardian.js") || !command.includes(
|
|
1925
|
+
if (!command.includes("process-guardian.js") || !command.includes(identity.socketPath)) throw new Error(`Refusing to stop unverified fixture guardian pid ${identity.pid}`)
|
|
1822
1926
|
const client = new GuardianClient(identity)
|
|
1823
1927
|
|
|
1824
1928
|
await client.connect()
|