rollbridge 0.1.58 → 0.1.60
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +20 -0
- package/changelog.d/20261106-release-dependency-boot-budgets.md +32 -0
- package/docs/config.md +40 -0
- package/docs/troubleshooting.md +43 -6
- package/package.json +1 -1
- package/src/config.js +91 -2
- package/src/daemon.js +10 -8
- package/src/guardian-client.js +13 -2
- package/src/health.js +99 -8
- package/src/process-guardian.js +125 -15
- package/src/process-order.js +26 -0
- package/src/release-group.js +157 -22
- package/test/config-validation.test.js +53 -0
- package/test/fixtures/deferred-boot-app.js +106 -0
- package/test/guardian-client.test.js +32 -0
- package/test/health.test.js +74 -1
- package/test/rollbridge.test.js +128 -0
|
@@ -58,6 +58,38 @@ test("guardian inventory removes only an exact owned provenance", async () => {
|
|
|
58
58
|
}
|
|
59
59
|
})
|
|
60
60
|
|
|
61
|
+
test("a refused owner mutation names the held mutation, its process, its age and its blocker", async () => {
|
|
62
|
+
const fixture = await createGuardian()
|
|
63
|
+
|
|
64
|
+
try {
|
|
65
|
+
const mutationId = await fixture.client.beginOwnerMutation("restart", {ownerPid: process.pid, waitingFor: "the managed process to stop and start again"})
|
|
66
|
+
await expect(fixture.client.beginOwnerMutation("deploy")).rejects.toThrow(/^Another owner mutation is already in progress: owner mutation "restart"; mutation [0-9a-f-]{36}; held by process \d+; since 20\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\.\d\d\dZ; \d+s elapsed; waiting for the managed process to stop and start again; refused deploy$/)
|
|
67
|
+
await fixture.client.reportOwnerMutationWait(mutationId, "an idle forked runner that never exits")
|
|
68
|
+
await expect(fixture.client.beginOwnerMutation("deploy")).rejects.toThrow(/waiting for an idle forked runner that never exits/)
|
|
69
|
+
await fixture.client.endOwnerMutation(mutationId)
|
|
70
|
+
expect(typeof await fixture.client.beginOwnerMutation("deploy")).toBe("string")
|
|
71
|
+
} finally {
|
|
72
|
+
await cleanupGuardian(fixture)
|
|
73
|
+
}
|
|
74
|
+
})
|
|
75
|
+
|
|
76
|
+
test("an owner mutation that outruns its lease is released and the holder reports the expiry", async () => {
|
|
77
|
+
const fixture = await createGuardian()
|
|
78
|
+
|
|
79
|
+
try {
|
|
80
|
+
const wedged = await fixture.client.beginOwnerMutation("restart", {mutationTimeoutMs: 200, ownerPid: process.pid, waitingFor: "a drain that never returns"})
|
|
81
|
+
|
|
82
|
+
await new Promise((resolve) => setTimeout(resolve, 600))
|
|
83
|
+
const successor = await fixture.client.beginOwnerMutation("deploy", {ownerPid: process.pid, waitingFor: "candidate activation"})
|
|
84
|
+
|
|
85
|
+
expect(typeof successor).toBe("string")
|
|
86
|
+
await expect(fixture.client.endOwnerMutation(wedged)).rejects.toThrow(/^Owner mutation restart exceeded its 200 ms lease and was released after \d+ ms; the mutation failed and its lease is gone$/)
|
|
87
|
+
await fixture.client.endOwnerMutation(successor)
|
|
88
|
+
} finally {
|
|
89
|
+
await cleanupGuardian(fixture)
|
|
90
|
+
}
|
|
91
|
+
})
|
|
92
|
+
|
|
61
93
|
test("guardian lets the owner re-register a key whose registered child is gone, and still fences a live one", async () => {
|
|
62
94
|
const fixture = await createGuardian()
|
|
63
95
|
const liveInstance = fixture.client.process("live-generation", definition("live-generation"))
|
package/test/health.test.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
3
|
import http from "node:http"
|
|
4
|
+
import net from "node:net"
|
|
4
5
|
import {describe, expect, test} from "@velocious/testing"
|
|
5
|
-
import {waitForHealth} from "../src/health.js"
|
|
6
|
+
import {waitForHealth, waitForListeningPort} from "../src/health.js"
|
|
6
7
|
|
|
7
8
|
describe("health", () => {
|
|
8
9
|
|
|
@@ -31,6 +32,18 @@ async function startHealthServer() {
|
|
|
31
32
|
}
|
|
32
33
|
}
|
|
33
34
|
|
|
35
|
+
/**
|
|
36
|
+
* @param {net.Server} server - Server to bind.
|
|
37
|
+
* @returns {Promise<number>} Ephemeral port it bound.
|
|
38
|
+
*/
|
|
39
|
+
async function bindPort(server) {
|
|
40
|
+
await new Promise((resolve) => server.listen(0, "127.0.0.1", () => resolve(undefined)))
|
|
41
|
+
|
|
42
|
+
const address = server.address()
|
|
43
|
+
|
|
44
|
+
return address && typeof address === "object" ? address.port : 0
|
|
45
|
+
}
|
|
46
|
+
|
|
34
47
|
test("waitForHealth delays the first probe by startDelayMs", async () => {
|
|
35
48
|
const server = await startHealthServer()
|
|
36
49
|
|
|
@@ -62,4 +75,64 @@ test("waitForHealth probes immediately when startDelayMs is 0", async () => {
|
|
|
62
75
|
await server.close()
|
|
63
76
|
}
|
|
64
77
|
})
|
|
78
|
+
|
|
79
|
+
test("waitForHealth names the deadline and the time it spent when the app never becomes healthy", async () => {
|
|
80
|
+
const server = http.createServer((_request, response) => {
|
|
81
|
+
response.writeHead(500)
|
|
82
|
+
response.end("bad")
|
|
83
|
+
})
|
|
84
|
+
|
|
85
|
+
try {
|
|
86
|
+
const port = await bindPort(server)
|
|
87
|
+
|
|
88
|
+
/** @type {import("../src/health.js").HealthCheckError} */
|
|
89
|
+
const failure = /** @type {import("../src/health.js").HealthCheckError} */ (await waitForHealth({
|
|
90
|
+
health: {intervalMs: 25, path: "/ping", startDelayMs: 0, timeoutMs: 300},
|
|
91
|
+
host: "127.0.0.1",
|
|
92
|
+
port,
|
|
93
|
+
processId: "web"
|
|
94
|
+
}).catch((error) => error instanceof Error ? error : new Error(String(error))))
|
|
95
|
+
|
|
96
|
+
expect(String(failure.message)).toMatch(/process web: \d+ probes, \d+ms elapsed of the 300ms health deadline/)
|
|
97
|
+
expect(failure.healthCheck?.deadlineMs).toBe(300)
|
|
98
|
+
expect({value: Boolean(failure.healthCheck && failure.healthCheck.elapsedMs >= 300), context: `expected the health gate to have spent its whole budget, was ${failure.healthCheck?.elapsedMs}ms`}).toMatchObject({value: true})
|
|
99
|
+
expect({value: Boolean(failure.healthCheck && failure.healthCheck.attempts >= 1), context: `expected at least one probe, was ${failure.healthCheck?.attempts}`}).toMatchObject({value: true})
|
|
100
|
+
expect(failure.healthCheck?.processId).toBe("web")
|
|
101
|
+
} finally {
|
|
102
|
+
await server.close()
|
|
103
|
+
}
|
|
104
|
+
})
|
|
105
|
+
|
|
106
|
+
test("waitForListeningPort waits for a dependency port on a budget of its own", async () => {
|
|
107
|
+
const server = net.createServer()
|
|
108
|
+
const port = await bindPort(server)
|
|
109
|
+
|
|
110
|
+
await new Promise((resolve) => server.close(() => resolve(undefined)))
|
|
111
|
+
|
|
112
|
+
const waiting = waitForListeningPort({host: "127.0.0.1", intervalMs: 25, port, timeoutMs: 2000})
|
|
113
|
+
|
|
114
|
+
await new Promise((resolve) => setTimeout(resolve, 300))
|
|
115
|
+
await new Promise((resolve) => server.listen(port, "127.0.0.1", () => resolve(undefined)))
|
|
116
|
+
|
|
117
|
+
try {
|
|
118
|
+
const result = await waiting
|
|
119
|
+
|
|
120
|
+
expect({value: Boolean(result.attempts >= 2), context: `expected more than one connection attempt, was ${result.attempts}`}).toMatchObject({value: true})
|
|
121
|
+
expect({value: Boolean(result.elapsedMs >= 250), context: `expected the dependency wait to take ~300ms, was ${result.elapsedMs}ms`}).toMatchObject({value: true})
|
|
122
|
+
} finally {
|
|
123
|
+
await new Promise((resolve) => server.close(() => resolve(undefined)))
|
|
124
|
+
}
|
|
125
|
+
})
|
|
126
|
+
|
|
127
|
+
test("waitForListeningPort names the budget it ran out on when the dependency never binds", async () => {
|
|
128
|
+
const server = net.createServer()
|
|
129
|
+
const port = await bindPort(server)
|
|
130
|
+
|
|
131
|
+
await new Promise((resolve) => server.close(() => resolve(undefined)))
|
|
132
|
+
|
|
133
|
+
/** @type {Error} */
|
|
134
|
+
const failure = /** @type {Error} */ (await waitForListeningPort({host: "127.0.0.1", intervalMs: 25, port, timeoutMs: 200}).catch((error) => error instanceof Error ? error : new Error(String(error))))
|
|
135
|
+
|
|
136
|
+
expect(String(failure.message)).toMatch(/did not accept a connection within 200ms/)
|
|
137
|
+
})
|
|
65
138
|
})
|
package/test/rollbridge.test.js
CHANGED
|
@@ -19,6 +19,7 @@ describe("rollbridge", () => {
|
|
|
19
19
|
const currentDir = path.dirname(fileURLToPath(import.meta.url))
|
|
20
20
|
const binPath = path.join(currentDir, "..", "bin", "rollbridge")
|
|
21
21
|
const dependentAppPath = path.join(currentDir, "fixtures", "dependent-app.js")
|
|
22
|
+
const deferredBootAppPath = path.join(currentDir, "fixtures", "deferred-boot-app.js")
|
|
22
23
|
const dummyAppPath = path.join(currentDir, "fixtures", "dummy-app.js")
|
|
23
24
|
const memoryHogPath = path.join(currentDir, "fixtures", "memory-hog.js")
|
|
24
25
|
const serviceAppPath = path.join(currentDir, "fixtures", "service-app.js")
|
|
@@ -243,6 +244,98 @@ test("failed release startup logs process output and cleanup status", async () =
|
|
|
243
244
|
}
|
|
244
245
|
})
|
|
245
246
|
|
|
247
|
+
test("the dependency wait is charged to its own budget, not to the proxied boot budget", async () => {
|
|
248
|
+
const fixture = await createFixture()
|
|
249
|
+
const config = deferredBootConfig(fixture, {dependencyTimeoutMs: 5000, serviceBootMs: 1200, webBootMs: 600, webHealthTimeoutMs: 900})
|
|
250
|
+
/** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
|
|
251
|
+
const logs = []
|
|
252
|
+
const daemon = new RollbridgeDaemon({config, logger: (message, data = {}) => logs.push({data, message})})
|
|
253
|
+
|
|
254
|
+
await daemon.start()
|
|
255
|
+
|
|
256
|
+
try {
|
|
257
|
+
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
258
|
+
|
|
259
|
+
const dependencyLog = logs.find((entry) => entry.message === "release process dependency ready")
|
|
260
|
+
const healthLog = logs.find((entry) => entry.message === "release candidate health ready")
|
|
261
|
+
|
|
262
|
+
if (!dependencyLog?.data || !healthLog?.data) throw new Error("expected the dependency wait and the boot time to be logged")
|
|
263
|
+
// The dependency took ~1200ms and the app boot ~600ms. The boot budget is 900ms, so this
|
|
264
|
+
// deploy only succeeds because the dependency wait is charged to a budget of its own.
|
|
265
|
+
expect(Number(dependencyLog.data.elapsedMs)).toBeGreaterThanOrEqual(1000)
|
|
266
|
+
expect(Number(healthLog.data.bootMs)).toBeLessThan(900)
|
|
267
|
+
expect(Number(healthLog.data.waitedForDependenciesMs)).toBeGreaterThanOrEqual(1000)
|
|
268
|
+
expect(await fetchText(daemon, "/release")).toBe("v1")
|
|
269
|
+
} finally {
|
|
270
|
+
await daemon.shutdown()
|
|
271
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
272
|
+
}
|
|
273
|
+
})
|
|
274
|
+
|
|
275
|
+
test("a dependency that never opens fails the deploy on its own budget and keeps the incumbent serving", async () => {
|
|
276
|
+
const fixture = await createFixture()
|
|
277
|
+
const initialConfig = deferredBootConfig(fixture, {dependencyTimeoutMs: 5000, serviceBootMs: 0, webBootMs: 0, webHealthTimeoutMs: 900})
|
|
278
|
+
const configPath = await writeConfigFile(initialConfig, fixture.root)
|
|
279
|
+
/** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
|
|
280
|
+
const logs = []
|
|
281
|
+
const daemon = new RollbridgeDaemon({config: initialConfig, configPath, logger: (message, data = {}) => logs.push({data, message})})
|
|
282
|
+
|
|
283
|
+
await daemon.start()
|
|
284
|
+
|
|
285
|
+
try {
|
|
286
|
+
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
287
|
+
expect(await fetchText(daemon, "/release")).toBe("v1")
|
|
288
|
+
|
|
289
|
+
// v2's service never binds its port, so the dependency gate runs out before the app is started.
|
|
290
|
+
await writeConfigFile(deferredBootConfig(fixture, {dependencyTimeoutMs: 300, serviceBootMs: 60000, webBootMs: 0, webHealthTimeoutMs: 900}), fixture.root)
|
|
291
|
+
await expect(daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})).rejects.toThrow(/did not accept a connection within 300ms/)
|
|
292
|
+
|
|
293
|
+
expect(daemon.status().activeReleaseId).toBe("v1")
|
|
294
|
+
expect(await fetchText(daemon, "/release")).toBe("v1")
|
|
295
|
+
|
|
296
|
+
const failure = logs.find((entry) => entry.message === "release startup failed" && entry.data?.phase === "before cleanup")
|
|
297
|
+
|
|
298
|
+
if (!failure?.data) throw new Error("expected the dependency failure to name its budget")
|
|
299
|
+
expect(failure.data.stage).toBe("dependency")
|
|
300
|
+
expect(failure.data.deadlineMs).toBe(300)
|
|
301
|
+
expect(Number(failure.data.dependencyWaitMs)).toBeGreaterThanOrEqual(300)
|
|
302
|
+
expect(failure.data.readyProcesses).toEqual(["beacon"])
|
|
303
|
+
expect(failure.data.unreadyProcesses).toEqual([])
|
|
304
|
+
} finally {
|
|
305
|
+
await daemon.shutdown()
|
|
306
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
307
|
+
}
|
|
308
|
+
})
|
|
309
|
+
|
|
310
|
+
test("a startup timeout names the deadline, the elapsed boot, the ready processes, and the last output", async () => {
|
|
311
|
+
const fixture = await createFixture()
|
|
312
|
+
const config = deferredBootConfig(fixture, {dependencyTimeoutMs: 5000, serviceBootMs: 0, webBootMs: 1200, webHealthTimeoutMs: 400})
|
|
313
|
+
/** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
|
|
314
|
+
const logs = []
|
|
315
|
+
const daemon = new RollbridgeDaemon({config, logger: (message, data = {}) => logs.push({data, message})})
|
|
316
|
+
|
|
317
|
+
await daemon.start()
|
|
318
|
+
|
|
319
|
+
try {
|
|
320
|
+
await expect(daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})).rejects.toThrow(/health deadline/)
|
|
321
|
+
|
|
322
|
+
const failure = logs.find((entry) => entry.message === "release startup failed" && entry.data?.phase === "before cleanup")
|
|
323
|
+
|
|
324
|
+
if (!failure?.data) throw new Error("expected the startup failure to name its budget")
|
|
325
|
+
expect(failure.data.stage).toBe("health")
|
|
326
|
+
expect(failure.data.deadlineMs).toBe(400)
|
|
327
|
+
expect(Number(failure.data.elapsedMs)).toBeGreaterThanOrEqual(400)
|
|
328
|
+
expect(Number(failure.data.probes)).toBeGreaterThanOrEqual(1)
|
|
329
|
+
expect(failure.data.readyProcesses).toEqual(["beacon"])
|
|
330
|
+
expect(failure.data.unreadyProcesses).toEqual(["web"])
|
|
331
|
+
expect(String(failure.data.lastOutput)).toMatch(/accepted a connection/)
|
|
332
|
+
expect(String(failure.data.lastOutputAt)).toMatch(/^\d{4}-\d{2}-\d{2}T/)
|
|
333
|
+
} finally {
|
|
334
|
+
await daemon.shutdown()
|
|
335
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
336
|
+
}
|
|
337
|
+
})
|
|
338
|
+
|
|
246
339
|
test("singleton processes restart without overlap during deploy", async () => {
|
|
247
340
|
const fixture = await createFixture({includeSingleton: true})
|
|
248
341
|
const daemon = await startDaemon(fixture.config)
|
|
@@ -2199,6 +2292,41 @@ async function createFixture(options = {}) {
|
|
|
2199
2292
|
return {activationGatePath, config, gatePath, lifecycleLogPath, retirementGatePath, root, serviceLogPath, serviceQuietPath, singletonLogPath, statePath}
|
|
2200
2293
|
}
|
|
2201
2294
|
|
|
2295
|
+
/**
|
|
2296
|
+
* Config where both processes run the deferred-boot fixture, so a measured boot budget can be
|
|
2297
|
+
* compared against a dependency wait that must not be charged to it.
|
|
2298
|
+
* @param {{config: import("../src/config.js").RollbridgeConfig, root: string}} fixture - Fixture.
|
|
2299
|
+
* @param {{dependencyTimeoutMs: number, serviceBootMs: number, webBootMs: number, webHealthTimeoutMs: number}} options - Budgets.
|
|
2300
|
+
* @returns {import("../src/config.js").RollbridgeConfig} Config for the deferred-boot processes.
|
|
2301
|
+
*/
|
|
2302
|
+
function deferredBootConfig(fixture, options) {
|
|
2303
|
+
const appCommand = `${JSON.stringify(process.execPath)} ${JSON.stringify(deferredBootAppPath)}`
|
|
2304
|
+
|
|
2305
|
+
return normalizeConfig({
|
|
2306
|
+
...fixture.config,
|
|
2307
|
+
processes: [
|
|
2308
|
+
{
|
|
2309
|
+
command: appCommand,
|
|
2310
|
+
deployStrategy: "handoff",
|
|
2311
|
+
env: {ROLLBRIDGE_BOOT_MS: String(options.serviceBootMs)},
|
|
2312
|
+
id: "beacon",
|
|
2313
|
+
policy: "service",
|
|
2314
|
+
port: {from: 15000, to: 15099}
|
|
2315
|
+
},
|
|
2316
|
+
{
|
|
2317
|
+
command: appCommand,
|
|
2318
|
+
dependsOn: ["beacon"],
|
|
2319
|
+
dependencyTimeoutMs: options.dependencyTimeoutMs,
|
|
2320
|
+
env: {ROLLBRIDGE_BOOT_MS: String(options.webBootMs), ROLLBRIDGE_DEPENDENCY_PORT: "{{ports.beacon}}"},
|
|
2321
|
+
health: {intervalMs: 50, path: "/ping", timeoutMs: options.webHealthTimeoutMs},
|
|
2322
|
+
id: "web",
|
|
2323
|
+
policy: "proxied",
|
|
2324
|
+
port: {from: 0, to: 0}
|
|
2325
|
+
}
|
|
2326
|
+
]
|
|
2327
|
+
})
|
|
2328
|
+
}
|
|
2329
|
+
|
|
2202
2330
|
/**
|
|
2203
2331
|
* @param {string} lifecycleLogPath - Fixture lifecycle event file.
|
|
2204
2332
|
* @returns {Promise<string[]>} Ordered lifecycle events.
|