rollbridge 0.1.21 → 0.1.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/cli.js +28 -3
- package/src/daemon.js +9 -1
- package/src/managed-process.js +52 -4
- package/test/daemon-bootstrap.test.js +198 -19
- package/test/fixtures/owned-child.js +19 -0
- package/test/managed-process.test.js +44 -8
package/package.json
CHANGED
package/src/cli.js
CHANGED
|
@@ -65,9 +65,19 @@ export async function runCli(argv) {
|
|
|
65
65
|
} else {
|
|
66
66
|
await daemon.exposeControl()
|
|
67
67
|
}
|
|
68
|
-
} catch {
|
|
69
|
-
|
|
70
|
-
|
|
68
|
+
} catch (error) {
|
|
69
|
+
const failure = error instanceof Error ? error : String(error)
|
|
70
|
+
|
|
71
|
+
daemon.logger("bootstrap activation failed", {releaseId: bootstrap.releaseId, status: "error", ...errorLogData(failure)})
|
|
72
|
+
|
|
73
|
+
try {
|
|
74
|
+
await daemon.shutdown()
|
|
75
|
+
} catch (shutdownError) {
|
|
76
|
+
const shutdownFailure = shutdownError instanceof Error ? shutdownError : String(shutdownError)
|
|
77
|
+
|
|
78
|
+
daemon.logger("bootstrap shutdown failed", {releaseId: bootstrap.releaseId, status: "error", ...errorLogData(shutdownFailure)})
|
|
79
|
+
}
|
|
80
|
+
|
|
71
81
|
process.exitCode = 1
|
|
72
82
|
return
|
|
73
83
|
}
|
|
@@ -960,3 +970,18 @@ function isMissingDaemonError(error) {
|
|
|
960
970
|
|
|
961
971
|
return error.code === "ENOENT" || error.code === "ECONNREFUSED"
|
|
962
972
|
}
|
|
973
|
+
|
|
974
|
+
/**
|
|
975
|
+
* Converts any thrown value into JSON-safe diagnostics without losing an Error stack.
|
|
976
|
+
* @param {Error | string} error - Thrown value.
|
|
977
|
+
* @returns {{error: string, stack: string}} Safe structured log fields.
|
|
978
|
+
*/
|
|
979
|
+
function errorLogData(error) {
|
|
980
|
+
if (error instanceof Error) {
|
|
981
|
+
return {error: error.message, stack: error.stack || `${error.name}: ${error.message}`}
|
|
982
|
+
}
|
|
983
|
+
|
|
984
|
+
const message = String(error)
|
|
985
|
+
|
|
986
|
+
return {error: message, stack: message}
|
|
987
|
+
}
|
package/src/daemon.js
CHANGED
|
@@ -56,6 +56,7 @@ export default class RollbridgeDaemon {
|
|
|
56
56
|
this.proxy = httpProxy.createProxyServer({ws: true, xfwd: true})
|
|
57
57
|
this.proxyServer = /** @type {http.Server | undefined} */ (undefined)
|
|
58
58
|
this.controlServer = /** @type {net.Server | undefined} */ (undefined)
|
|
59
|
+
this.controlSocketOwned = false
|
|
59
60
|
this.controlSockets = /** @type {Set<net.Socket>} */ (new Set())
|
|
60
61
|
this.proxyPort = /** @type {number | undefined} */ (undefined)
|
|
61
62
|
this.stopping = false
|
|
@@ -63,6 +64,7 @@ export default class RollbridgeDaemon {
|
|
|
63
64
|
this.persistTimer = /** @type {ReturnType<typeof setInterval> | undefined} */ (undefined)
|
|
64
65
|
this.persistenceEnabled = false
|
|
65
66
|
this.pendingWrite = /** @type {Promise<void> | undefined} */ (undefined)
|
|
67
|
+
this.stateCleanupEnabled = false
|
|
66
68
|
this.shutdownPromise = /** @type {Promise<void> | undefined} */ (undefined)
|
|
67
69
|
this.retirementPromise = /** @type {Promise<void> | undefined} */ (undefined)
|
|
68
70
|
this.controlClosePromise = /** @type {Promise<void> | undefined} */ (undefined)
|
|
@@ -128,6 +130,7 @@ export default class RollbridgeDaemon {
|
|
|
128
130
|
await new Promise((resolve, reject) => {
|
|
129
131
|
server.once("error", reject)
|
|
130
132
|
server.listen(this.config.control.path, () => {
|
|
133
|
+
this.controlSocketOwned = true
|
|
131
134
|
this.logger("control socket listening", {path: this.config.control.path})
|
|
132
135
|
resolve(undefined)
|
|
133
136
|
})
|
|
@@ -741,6 +744,7 @@ export default class RollbridgeDaemon {
|
|
|
741
744
|
startStatePersistence() {
|
|
742
745
|
if (!this.statePath) return
|
|
743
746
|
|
|
747
|
+
this.stateCleanupEnabled = true
|
|
744
748
|
this.persistenceEnabled = true
|
|
745
749
|
this.persistState()
|
|
746
750
|
this.persistTimer = setInterval(() => this.persistState(), STATE_PERSIST_INTERVAL_MS)
|
|
@@ -780,6 +784,7 @@ export default class RollbridgeDaemon {
|
|
|
780
784
|
async reportOrphans() {
|
|
781
785
|
if (!this.statePath) return
|
|
782
786
|
|
|
787
|
+
this.stateCleanupEnabled = true
|
|
783
788
|
const orphans = liveProcesses(await readState(this.statePath))
|
|
784
789
|
|
|
785
790
|
// Keep them for status() so `rollbridge status` reflects still-running children after a
|
|
@@ -888,7 +893,7 @@ export default class RollbridgeDaemon {
|
|
|
888
893
|
// new writes start: stopping is set and the persist timer is cleared above). Prior-daemon
|
|
889
894
|
// orphans are not owned by this daemon, so retain their records until they are confirmed gone.
|
|
890
895
|
await captureShutdownError(cleanupErrors, "persistent state cleanup", async () => {
|
|
891
|
-
if (!this.statePath) return
|
|
896
|
+
if (!this.statePath || !this.stateCleanupEnabled) return
|
|
892
897
|
if (this.pendingWrite) await this.pendingWrite
|
|
893
898
|
const orphans = this.orphans.filter((orphan) => isProcessAlive(orphan.pid))
|
|
894
899
|
|
|
@@ -910,7 +915,10 @@ export default class RollbridgeDaemon {
|
|
|
910
915
|
|
|
911
916
|
/** @returns {Promise<void>} Removes the configured control socket path. */
|
|
912
917
|
async removeControlSocket() {
|
|
918
|
+
if (!this.controlSocketOwned) return
|
|
919
|
+
|
|
913
920
|
await fs.rm(this.config.control.path, {force: true})
|
|
921
|
+
this.controlSocketOwned = false
|
|
914
922
|
}
|
|
915
923
|
|
|
916
924
|
/**
|
package/src/managed-process.js
CHANGED
|
@@ -62,6 +62,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
62
62
|
this.memoryWarned = false
|
|
63
63
|
this.startedAtMs = /** @type {number | undefined} */ (undefined)
|
|
64
64
|
this.intentionalStop = false
|
|
65
|
+
this.intentionalStopSignal = /** @type {ProcessExitSignal | undefined} */ (undefined)
|
|
65
66
|
this.quiescePromise = /** @type {Promise<void> | undefined} */ (undefined)
|
|
66
67
|
this.restartTimer = undefined
|
|
67
68
|
this.child = undefined
|
|
@@ -79,6 +80,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
79
80
|
if (this.child) return
|
|
80
81
|
|
|
81
82
|
this.intentionalStop = false
|
|
83
|
+
this.intentionalStopSignal = undefined
|
|
82
84
|
this.quiescePromise = undefined
|
|
83
85
|
this.exitCode = undefined
|
|
84
86
|
this.exitSignal = undefined
|
|
@@ -167,8 +169,8 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
167
169
|
onExit(code, signal) {
|
|
168
170
|
const wasIntentional = this.intentionalStop
|
|
169
171
|
|
|
170
|
-
this.exitCode = code
|
|
171
|
-
this.exitSignal = signal
|
|
172
|
+
this.exitCode = this.intentionalStopSignal ? null : code
|
|
173
|
+
this.exitSignal = signal ?? this.intentionalStopSignal
|
|
172
174
|
this.child = undefined
|
|
173
175
|
this.pid = undefined
|
|
174
176
|
this.exitPromise = undefined
|
|
@@ -363,13 +365,16 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
363
365
|
// 3. Stop whatever is still running, then SIGKILL if it outlasts the graceful window.
|
|
364
366
|
if (this.processGroupExists(pgid)) {
|
|
365
367
|
if (stopCommand) await this.runHook(stopCommand, hookTimeoutMs, "stop command", pgid)
|
|
366
|
-
else
|
|
368
|
+
else {
|
|
369
|
+
this.intentionalStopSignal = /** @type {ProcessExitSignal} */ (this.stopSignal)
|
|
370
|
+
await this.signalProcessGroup(this.stopSignal, pgid, options.timeoutMs ?? this.stopTimeoutMs)
|
|
371
|
+
}
|
|
367
372
|
|
|
368
373
|
const timeoutMs = options.timeoutMs ?? this.stopTimeoutMs
|
|
369
374
|
|
|
370
375
|
if (!(await this.waitForProcessGroupExit(pgid, timeoutMs))) {
|
|
371
376
|
this.logger("process stop timed out; sending SIGKILL", {id: this.id, pid: pgid})
|
|
372
|
-
this.
|
|
377
|
+
await this.signalProcessGroup("SIGKILL", pgid, 5000)
|
|
373
378
|
await this.waitForProcessGroupExit(pgid, 5000)
|
|
374
379
|
}
|
|
375
380
|
}
|
|
@@ -489,6 +494,49 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
489
494
|
}
|
|
490
495
|
}
|
|
491
496
|
|
|
497
|
+
/**
|
|
498
|
+
* Signals descendants before their shell leader so the leader can reap them before it exits.
|
|
499
|
+
* Falls back to the portable group signal when procfs cannot identify group members.
|
|
500
|
+
* @param {string} signal - Signal name.
|
|
501
|
+
* @param {number} pgid - Process group id.
|
|
502
|
+
* @param {StopTimeoutMs} timeoutMs - Maximum time to wait for descendant reaping.
|
|
503
|
+
* @returns {Promise<void>} Resolves after the leader has been signalled.
|
|
504
|
+
*/
|
|
505
|
+
async signalProcessGroup(signal, pgid, timeoutMs) {
|
|
506
|
+
const members = processGroupMembers(pgid)
|
|
507
|
+
const descendants = members.filter((member) => member.pid !== pgid)
|
|
508
|
+
|
|
509
|
+
if (descendants.length === 0) {
|
|
510
|
+
this.killProcessGroup(signal, pgid)
|
|
511
|
+
return
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
for (const descendant of descendants) this.killProcess(descendant.pid, signal)
|
|
515
|
+
const deadline = timeoutMs === "indefinite" ? undefined : Date.now() + timeoutMs
|
|
516
|
+
|
|
517
|
+
while (processGroupMembers(pgid).some((member) => member.pid !== pgid)) {
|
|
518
|
+
if (deadline !== undefined && Date.now() >= deadline) break
|
|
519
|
+
await new Promise((resolve) => setTimeout(resolve, 25))
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
this.killProcess(pgid, signal)
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
/**
|
|
526
|
+
* Signals one verified member of the owned process group.
|
|
527
|
+
* @param {number} pid - Process id.
|
|
528
|
+
* @param {string} signal - Signal name.
|
|
529
|
+
* @returns {void}
|
|
530
|
+
*/
|
|
531
|
+
killProcess(pid, signal) {
|
|
532
|
+
try {
|
|
533
|
+
process.kill(pid, signal)
|
|
534
|
+
} catch (error) {
|
|
535
|
+
if (error && typeof error === "object" && "code" in error && error.code === "ESRCH") return
|
|
536
|
+
throw error
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
|
|
492
540
|
/**
|
|
493
541
|
* @param {number} pgid - Process group id.
|
|
494
542
|
* @returns {boolean} True until the process group no longer exists.
|
|
@@ -4,6 +4,7 @@ import assert from "node:assert/strict"
|
|
|
4
4
|
import {spawn} from "node:child_process"
|
|
5
5
|
import {once} from "node:events"
|
|
6
6
|
import fs from "node:fs/promises"
|
|
7
|
+
import net from "node:net"
|
|
7
8
|
import os from "node:os"
|
|
8
9
|
import path from "node:path"
|
|
9
10
|
import test from "node:test"
|
|
@@ -14,9 +15,12 @@ import {isProcessAlive, liveProcesses, readState, writeState} from "../src/state
|
|
|
14
15
|
const currentDir = path.dirname(fileURLToPath(import.meta.url))
|
|
15
16
|
const binPath = path.join(currentDir, "..", "bin", "rollbridge")
|
|
16
17
|
const dummyAppPath = path.join(currentDir, "fixtures", "dummy-app.js")
|
|
18
|
+
const ownedChildPath = path.join(currentDir, "fixtures", "owned-child.js")
|
|
17
19
|
const firstAttestation = `sha256:${"a".repeat(64)}`
|
|
18
20
|
const secondAttestation = `sha256:${"b".repeat(64)}`
|
|
19
21
|
|
|
22
|
+
/** @typedef {{data?: Record<string, import("../src/json.js").JsonValue>, message?: string}} StructuredRecord */
|
|
23
|
+
|
|
20
24
|
test("daemon bootstrap requires complete, safe, absolute inputs before binding listeners", async (t) => {
|
|
21
25
|
const cases = [
|
|
22
26
|
{args: ["--config", "CONFIG", "--release-path", "RELEASE", "--release-id", "v1"], message: /must be provided together/},
|
|
@@ -86,11 +90,12 @@ test("daemon bootstrap activates the exact release through the foreground daemon
|
|
|
86
90
|
})
|
|
87
91
|
|
|
88
92
|
test("failed takeover bootstrap preserves the previously accepted owner", async () => {
|
|
89
|
-
const fixture = await createFixture()
|
|
93
|
+
const fixture = await createFixture({persistState: true})
|
|
90
94
|
const accepted = spawnDaemon(fixture, {attestation: firstAttestation, releaseId: "accepted", revision: "accepted123"})
|
|
91
95
|
|
|
92
96
|
try {
|
|
93
97
|
await waitForLog(accepted, "control socket listening")
|
|
98
|
+
await waitForFile(fixture.statePath)
|
|
94
99
|
const badConfig = JSON.parse((await fs.readFile(fixture.configPath, "utf8")).replace(/^module\.exports = /, ""))
|
|
95
100
|
badConfig.processes[0].health.path = "/never-ready"
|
|
96
101
|
badConfig.processes[0].health.timeoutMs = 100
|
|
@@ -106,10 +111,20 @@ test("failed takeover bootstrap preserves the previously accepted owner", async
|
|
|
106
111
|
])
|
|
107
112
|
|
|
108
113
|
assert.notEqual(result.code, 0)
|
|
114
|
+
const records = parseStructuredOutput(result.output)
|
|
115
|
+
const failure = records.find((record) => record.message === "bootstrap activation failed")
|
|
116
|
+
const candidatePid = Number(await fs.readFile(fixture.startedPath, "utf8"))
|
|
117
|
+
|
|
118
|
+
assert.match(String(failure?.data?.error), /Health check failed/)
|
|
119
|
+
assert.match(String(failure?.data?.stack), /Error: Health check failed/)
|
|
120
|
+
assert.equal(isProcessAlive(candidatePid), false, "the failed candidate process must be stopped")
|
|
109
121
|
const status = await sendControlCommand({command: {command: "status"}, path: fixture.socketPath})
|
|
110
122
|
assert.equal(status.activeReleaseId, "accepted")
|
|
111
123
|
assert.ok(status.bootstrap && typeof status.bootstrap === "object" && !Array.isArray(status.bootstrap))
|
|
112
124
|
assert.equal(status.bootstrap.attestation, firstAttestation)
|
|
125
|
+
const priorState = await readState(fixture.statePath)
|
|
126
|
+
|
|
127
|
+
assert.ok(priorState && typeof priorState === "object" && !Array.isArray(priorState) && priorState.activeReleaseId === "accepted", "candidate cleanup must preserve the prior owner's state")
|
|
113
128
|
} finally {
|
|
114
129
|
accepted.kill("SIGTERM")
|
|
115
130
|
if (accepted.exitCode === null) await once(accepted, "exit")
|
|
@@ -267,24 +282,56 @@ test("SIGTERM during multi-process bootstrap owns every process started after sh
|
|
|
267
282
|
}
|
|
268
283
|
})
|
|
269
284
|
|
|
270
|
-
test("failed
|
|
271
|
-
const fixture = await createFixture({
|
|
285
|
+
test("failed ordinary bootstrap completely shuts down attempt-owned resources and exposes the cause", async () => {
|
|
286
|
+
const fixture = await createFixture({attemptOwnedProcesses: true, fixedPorts: true, missingControlParent: true})
|
|
272
287
|
|
|
273
288
|
try {
|
|
274
289
|
const result = await runDaemon([
|
|
275
290
|
"--config", fixture.configPath,
|
|
276
291
|
"--release-path", fixture.root,
|
|
277
|
-
"--release-id", "
|
|
278
|
-
"--revision", "
|
|
292
|
+
"--release-id", "ordinary-failure",
|
|
293
|
+
"--revision", "ordinary123"
|
|
279
294
|
])
|
|
280
|
-
const records = result.output
|
|
295
|
+
const records = parseStructuredOutput(result.output)
|
|
296
|
+
const failure = records.find((record) => record.message === "bootstrap activation failed")
|
|
297
|
+
|
|
298
|
+
assert.notEqual(result.code, 0)
|
|
299
|
+
assert.equal(failure?.data?.releaseId, "ordinary-failure")
|
|
300
|
+
assert.equal(failure?.data?.status, "error")
|
|
301
|
+
assert.match(String(failure?.data?.error), /listen (?:EACCES|ENOENT)/)
|
|
302
|
+
assert.match(String(failure?.data?.stack), /Error: listen (?:EACCES|ENOENT)/)
|
|
303
|
+
await assertAttemptResourcesStopped(fixture, records)
|
|
304
|
+
await assert.rejects(() => fs.stat(fixture.socketPath), {code: "ENOENT"})
|
|
305
|
+
} finally {
|
|
306
|
+
await killAttemptProcesses(fixture.lifecyclePath)
|
|
307
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
308
|
+
}
|
|
309
|
+
})
|
|
310
|
+
|
|
311
|
+
test("failed takeover retirement completely shuts down and exits non-zero instead of lingering", async () => {
|
|
312
|
+
const fixture = await createFixture({attemptOwnedProcesses: true, fixedPorts: true})
|
|
313
|
+
|
|
314
|
+
try {
|
|
315
|
+
const result = await runDaemon([
|
|
316
|
+
"--config", fixture.configPath,
|
|
317
|
+
"--release-path", fixture.root,
|
|
318
|
+
"--release-id", "orphaned-candidate",
|
|
319
|
+
"--revision", "candidate123",
|
|
320
|
+
"--boot-attestation", secondAttestation,
|
|
321
|
+
"--takeover-owner"
|
|
322
|
+
], {timeoutMs: 2000})
|
|
323
|
+
const records = parseStructuredOutput(result.output)
|
|
281
324
|
const failure = records.find((record) => record.message === "bootstrap activation failed")
|
|
282
325
|
|
|
283
326
|
assert.notEqual(result.code, 0)
|
|
284
|
-
assert.
|
|
285
|
-
assert.
|
|
327
|
+
assert.equal(failure?.data?.releaseId, "orphaned-candidate")
|
|
328
|
+
assert.equal(failure?.data?.status, "error")
|
|
329
|
+
assert.match(String(failure?.data?.error), /connect ENOENT/)
|
|
330
|
+
assert.match(String(failure?.data?.stack), /Error: connect ENOENT/)
|
|
331
|
+
await assertAttemptResourcesStopped(fixture, records)
|
|
286
332
|
await assert.rejects(() => fs.stat(fixture.socketPath), {code: "ENOENT"})
|
|
287
333
|
} finally {
|
|
334
|
+
await killAttemptProcesses(fixture.lifecyclePath)
|
|
288
335
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
289
336
|
}
|
|
290
337
|
})
|
|
@@ -350,12 +397,12 @@ test("failed daemon bootstrap preserves prior live process records in statePath"
|
|
|
350
397
|
})
|
|
351
398
|
|
|
352
399
|
/**
|
|
353
|
-
* @param {{healthGate?: boolean, healthPath?: string, healthTimeoutMs?: number, multiProcessSignal?: boolean, persistState?: boolean}} [options] - Fixture options.
|
|
354
|
-
* @returns {Promise<{configPath: string, gatePath: string, healthGatePath: string, lifecyclePath: string, root: string, socketPath: string, startedPath: string, statePath: string, stoppedPath: string}>} Fixture paths.
|
|
400
|
+
* @param {{attemptOwnedProcesses?: boolean, fixedPorts?: boolean, healthGate?: boolean, healthPath?: string, healthTimeoutMs?: number, missingControlParent?: boolean, multiProcessSignal?: boolean, persistState?: boolean}} [options] - Fixture options.
|
|
401
|
+
* @returns {Promise<{configPath: string, gatePath: string, healthGatePath: string, lifecyclePath: string, processPort: number, proxyPort: number, root: string, socketPath: string, startedPath: string, statePath: string, stoppedPath: string}>} Fixture paths.
|
|
355
402
|
*/
|
|
356
|
-
async function createFixture({healthGate = false, healthPath = "/ping", healthTimeoutMs = 1000, multiProcessSignal = false, persistState = false} = {}) {
|
|
403
|
+
async function createFixture({attemptOwnedProcesses = false, fixedPorts = false, healthGate = false, healthPath = "/ping", healthTimeoutMs = 1000, missingControlParent = false, multiProcessSignal = false, persistState = false} = {}) {
|
|
357
404
|
const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-bootstrap-"))
|
|
358
|
-
const socketPath = path.join(root, "control.sock")
|
|
405
|
+
const socketPath = missingControlParent ? path.join(root, "missing", "control.sock") : path.join(root, "control.sock")
|
|
359
406
|
const statePath = path.join(root, "state.json")
|
|
360
407
|
const startedPath = path.join(root, "started.pid")
|
|
361
408
|
const stoppedPath = path.join(root, "stopped.pid")
|
|
@@ -364,8 +411,12 @@ async function createFixture({healthGate = false, healthPath = "/ping", healthTi
|
|
|
364
411
|
const healthGatePath = path.join(root, "health-ready")
|
|
365
412
|
const configPath = path.join(root, "rollbridge.js")
|
|
366
413
|
const command = `${JSON.stringify(process.execPath)} ${JSON.stringify(dummyAppPath)}`
|
|
414
|
+
const ownedCommand = `exec ${JSON.stringify(process.execPath)} ${JSON.stringify(ownedChildPath)}`
|
|
415
|
+
const ownedWebCommand = `exec ${command}`
|
|
416
|
+
const [processPort, proxyPort] = fixedPorts ? await availablePorts(2) : [0, 0]
|
|
367
417
|
const lifecycleEnv = {ROLLBRIDGE_TEST_LIFECYCLE_PATH: lifecyclePath}
|
|
368
418
|
const webEnv = {
|
|
419
|
+
...(attemptOwnedProcesses ? lifecycleEnv : {}),
|
|
369
420
|
ROLLBRIDGE_TEST_STARTED_PATH: startedPath,
|
|
370
421
|
ROLLBRIDGE_TEST_STOPPED_PATH: stoppedPath,
|
|
371
422
|
...(healthGate ? {ROLLBRIDGE_TEST_HEALTH_GATE_PATH: healthGatePath} : {})
|
|
@@ -377,8 +428,13 @@ async function createFixture({healthGate = false, healthPath = "/ping", healthTi
|
|
|
377
428
|
{command: `trap '' TERM; printf '%s\\n' '{"event":"shutdown"}' >> ${JSON.stringify(lifecyclePath)}; kill -TERM "$ROLLBRIDGE_TEST_DAEMON_PID"; printf '{"event":"started","pid":%s,"processId":"database","replicaIndex":"0"}\\n' "$$" >> ${JSON.stringify(lifecyclePath)}; read ignored < ${JSON.stringify(gatePath)}`, env: lifecycleEnv, id: "database", policy: "service"},
|
|
378
429
|
{command: `exec ${command}`, env: lifecycleEnv, id: "worker", policy: "companion", replicas: 2},
|
|
379
430
|
{command: `exec ${command}`, env: lifecycleEnv, health: {path: healthPath, timeoutMs: healthTimeoutMs}, id: "web", policy: "proxied", port: {from: 0, to: 0}}
|
|
431
|
+
] : attemptOwnedProcesses ? [
|
|
432
|
+
{command: ownedCommand, env: lifecycleEnv, id: "database", policy: "service"},
|
|
433
|
+
{command: ownedCommand, env: lifecycleEnv, id: "worker", policy: "companion"},
|
|
434
|
+
{command: ownedCommand, env: lifecycleEnv, id: "scheduler", policy: "singleton"},
|
|
435
|
+
{command: ownedWebCommand, env: webEnv, health: {path: healthPath, timeoutMs: healthTimeoutMs}, id: "web", policy: "proxied", port: {from: processPort, to: processPort}}
|
|
380
436
|
] : [{command, env: webEnv, health: {path: healthPath, timeoutMs: healthTimeoutMs}, id: "web", policy: "proxied", port: {from: 0, to: 0}}],
|
|
381
|
-
proxy: {host: "127.0.0.1", port:
|
|
437
|
+
proxy: {host: "127.0.0.1", port: proxyPort},
|
|
382
438
|
...(persistState ? {statePath} : {})
|
|
383
439
|
}
|
|
384
440
|
|
|
@@ -392,7 +448,7 @@ async function createFixture({healthGate = false, healthPath = "/ping", healthTi
|
|
|
392
448
|
}
|
|
393
449
|
|
|
394
450
|
await fs.writeFile(configPath, `${setup}module.exports = ${JSON.stringify(config, null, 2)}\n`)
|
|
395
|
-
return {configPath, gatePath, healthGatePath, lifecyclePath, root, socketPath, startedPath, statePath, stoppedPath}
|
|
451
|
+
return {configPath, gatePath, healthGatePath, lifecyclePath, processPort, proxyPort, root, socketPath, startedPath, statePath, stoppedPath}
|
|
396
452
|
}
|
|
397
453
|
|
|
398
454
|
/**
|
|
@@ -427,6 +483,12 @@ async function waitForFile(filePath) {
|
|
|
427
483
|
const watcher = fs.watch(path.dirname(filePath))
|
|
428
484
|
|
|
429
485
|
try {
|
|
486
|
+
try {
|
|
487
|
+
return await fs.readFile(filePath, "utf8")
|
|
488
|
+
} catch (error) {
|
|
489
|
+
if (!error || typeof error !== "object" || !("code" in error) || error.code !== "ENOENT") throw error
|
|
490
|
+
}
|
|
491
|
+
|
|
430
492
|
for await (const event of watcher) {
|
|
431
493
|
if (event.filename === path.basename(filePath)) return await fs.readFile(filePath, "utf8")
|
|
432
494
|
}
|
|
@@ -452,26 +514,143 @@ function spawnDaemon(fixture, release) {
|
|
|
452
514
|
|
|
453
515
|
/**
|
|
454
516
|
* @param {string[]} args - Daemon arguments.
|
|
517
|
+
* @param {{timeoutMs?: number}} [options] - Process execution options.
|
|
455
518
|
* @returns {Promise<{code: number | null, output: string, stderr: string}>} Completed process result.
|
|
456
519
|
*/
|
|
457
|
-
async function runDaemon(args) {
|
|
458
|
-
return await runRollbridge(["daemon", ...args])
|
|
520
|
+
async function runDaemon(args, options) {
|
|
521
|
+
return await runRollbridge(["daemon", ...args], options)
|
|
459
522
|
}
|
|
460
523
|
|
|
461
524
|
/**
|
|
462
525
|
* @param {string[]} args - Rollbridge command and arguments.
|
|
526
|
+
* @param {{timeoutMs?: number}} [options] - Process execution options.
|
|
463
527
|
* @returns {Promise<{code: number | null, output: string, stderr: string}>} Completed process result.
|
|
464
528
|
*/
|
|
465
|
-
async function runRollbridge(args) {
|
|
529
|
+
async function runRollbridge(args, {timeoutMs} = {}) {
|
|
466
530
|
const child = spawn(process.execPath, [binPath, ...args], {stdio: ["ignore", "pipe", "pipe"]})
|
|
467
531
|
let output = ""
|
|
468
532
|
let stderr = ""
|
|
469
533
|
|
|
470
534
|
child.stdout.setEncoding("utf8").on("data", (chunk) => { output += chunk })
|
|
471
535
|
child.stderr.setEncoding("utf8").on("data", (chunk) => { stderr += chunk })
|
|
472
|
-
|
|
536
|
+
let timer
|
|
537
|
+
|
|
538
|
+
const exited = once(child, "exit")
|
|
539
|
+
const result = timeoutMs === undefined ? await exited : await Promise.race([
|
|
540
|
+
exited,
|
|
541
|
+
new Promise((resolve) => {
|
|
542
|
+
timer = setTimeout(() => {
|
|
543
|
+
child.kill("SIGKILL")
|
|
544
|
+
resolve(["timeout"])
|
|
545
|
+
}, timeoutMs)
|
|
546
|
+
})
|
|
547
|
+
])
|
|
548
|
+
|
|
549
|
+
if (timer) clearTimeout(timer)
|
|
550
|
+
if (result[0] === "timeout") {
|
|
551
|
+
await exited
|
|
552
|
+
throw new Error(`Rollbridge did not terminate within ${timeoutMs}ms`)
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
const [code] = result
|
|
556
|
+
|
|
557
|
+
return {code: typeof code === "number" || code === null ? code : null, output, stderr}
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
/**
|
|
561
|
+
* @param {string} output - JSON-lines daemon output.
|
|
562
|
+
* @returns {StructuredRecord[]} Parsed records.
|
|
563
|
+
*/
|
|
564
|
+
function parseStructuredOutput(output) {
|
|
565
|
+
return output.split("\n").filter(Boolean).map((line) => JSON.parse(line))
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
/**
|
|
569
|
+
* @param {{lifecyclePath: string, processPort: number, proxyPort: number}} fixture - Attempt fixture.
|
|
570
|
+
* @param {StructuredRecord[]} records - Structured daemon records.
|
|
571
|
+
* @returns {Promise<void>} Resolves after all owned resources are verified stopped.
|
|
572
|
+
*/
|
|
573
|
+
async function assertAttemptResourcesStopped(fixture, records) {
|
|
574
|
+
const events = await readLifecycleEvents(fixture.lifecyclePath)
|
|
575
|
+
const started = events.filter((event) => event.event === "started")
|
|
576
|
+
const stoppedPids = new Set(events.filter((event) => event.event === "stopped").map((event) => event.pid))
|
|
577
|
+
const expectedProcessIds = new Set(["database", "scheduler", "web", "worker"])
|
|
578
|
+
const managedStarts = new Set(records.filter((record) => record.message === "process started").map((record) => record.data?.processId))
|
|
579
|
+
const managedExits = new Set(records.filter((record) => record.message === "process exited").map((record) => record.data?.processId))
|
|
580
|
+
|
|
581
|
+
assert.deepEqual(managedStarts, expectedProcessIds)
|
|
582
|
+
assert.deepEqual(managedExits, expectedProcessIds)
|
|
583
|
+
for (const event of started) {
|
|
584
|
+
assert.equal(stoppedPids.has(event.pid), true, `${event.processId} must receive graceful shutdown`)
|
|
585
|
+
assert.equal(isProcessAlive(Number(event.pid)), false, `${event.processId} pid ${event.pid} must be gone before daemon exit`)
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
await assertPortAvailable(fixture.processPort)
|
|
589
|
+
await assertPortAvailable(fixture.proxyPort)
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
/**
|
|
593
|
+
* @param {string} lifecyclePath - Fixture lifecycle log.
|
|
594
|
+
* @returns {Promise<Record<string, import("../src/json.js").JsonValue>[]>} Parsed lifecycle events.
|
|
595
|
+
*/
|
|
596
|
+
async function readLifecycleEvents(lifecyclePath) {
|
|
597
|
+
return (await fs.readFile(lifecyclePath, "utf8")).trim().split("\n").filter(Boolean).map((line) => JSON.parse(line))
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
/**
|
|
601
|
+
* Best-effort cleanup for a deliberately failing lingering-process regression.
|
|
602
|
+
* @param {string} lifecyclePath - Fixture lifecycle log.
|
|
603
|
+
* @returns {Promise<void>} Resolves after known fixture process groups are stopped.
|
|
604
|
+
*/
|
|
605
|
+
async function killAttemptProcesses(lifecyclePath) {
|
|
606
|
+
try {
|
|
607
|
+
const events = await readLifecycleEvents(lifecyclePath)
|
|
608
|
+
|
|
609
|
+
for (const event of events) {
|
|
610
|
+
const pid = Number(event.pid)
|
|
611
|
+
|
|
612
|
+
if (event.event === "started" && isProcessAlive(pid)) process.kill(-pid, "SIGKILL")
|
|
613
|
+
}
|
|
614
|
+
} catch {
|
|
615
|
+
// The attempt can fail before creating the lifecycle file.
|
|
616
|
+
}
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
/**
|
|
620
|
+
* @param {number} port - Local port expected to be free.
|
|
621
|
+
* @returns {Promise<void>} Resolves after binding and releasing the port.
|
|
622
|
+
*/
|
|
623
|
+
async function assertPortAvailable(port) {
|
|
624
|
+
const server = net.createServer()
|
|
625
|
+
|
|
626
|
+
await new Promise((resolve, reject) => {
|
|
627
|
+
server.once("error", reject)
|
|
628
|
+
server.listen(port, "127.0.0.1", () => resolve(undefined))
|
|
629
|
+
})
|
|
630
|
+
await new Promise((resolve) => server.close(() => resolve(undefined)))
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
/**
|
|
634
|
+
* Reserves distinct ephemeral ports until all have been observed, then releases them.
|
|
635
|
+
* @param {number} count - Number of ports.
|
|
636
|
+
* @returns {Promise<number[]>} Available port numbers.
|
|
637
|
+
*/
|
|
638
|
+
async function availablePorts(count) {
|
|
639
|
+
const servers = Array.from({length: count}, () => net.createServer())
|
|
640
|
+
|
|
641
|
+
await Promise.all(servers.map((server) => new Promise((resolve, reject) => {
|
|
642
|
+
server.once("error", reject)
|
|
643
|
+
server.listen(0, "127.0.0.1", () => resolve(undefined))
|
|
644
|
+
})))
|
|
645
|
+
const ports = servers.map((server) => {
|
|
646
|
+
const address = server.address()
|
|
647
|
+
|
|
648
|
+
if (!address || typeof address === "string") throw new Error("Expected a TCP server address")
|
|
649
|
+
return address.port
|
|
650
|
+
})
|
|
473
651
|
|
|
474
|
-
|
|
652
|
+
await Promise.all(servers.map((server) => new Promise((resolve) => server.close(() => resolve(undefined)))))
|
|
653
|
+
return ports
|
|
475
654
|
}
|
|
476
655
|
|
|
477
656
|
/**
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
import fs from "node:fs"
|
|
4
|
+
|
|
5
|
+
const lifecyclePath = process.env.ROLLBRIDGE_TEST_LIFECYCLE_PATH
|
|
6
|
+
|
|
7
|
+
if (!lifecyclePath) throw new Error("ROLLBRIDGE_TEST_LIFECYCLE_PATH is required")
|
|
8
|
+
|
|
9
|
+
/** @param {string} event - Lifecycle event. */
|
|
10
|
+
const record = (event) => {
|
|
11
|
+
fs.appendFileSync(lifecyclePath, `${JSON.stringify({event, pid: process.pid, processId: process.env.ROLLBRIDGE_PROCESS_ID})}\n`)
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
record("started")
|
|
15
|
+
process.on("SIGTERM", () => {
|
|
16
|
+
record("stopped")
|
|
17
|
+
process.exit(0)
|
|
18
|
+
})
|
|
19
|
+
setInterval(() => {}, 1000)
|
|
@@ -401,18 +401,18 @@ test("sends the configured stopSignal as the graceful stop signal", async () =>
|
|
|
401
401
|
|
|
402
402
|
/** @type {string[]} */
|
|
403
403
|
const signals = []
|
|
404
|
-
const
|
|
404
|
+
const killProcess = managed.killProcess.bind(managed)
|
|
405
405
|
|
|
406
|
-
managed.
|
|
406
|
+
managed.killProcess = (pid, signal) => {
|
|
407
407
|
signals.push(signal)
|
|
408
|
-
|
|
408
|
+
killProcess(pid, signal)
|
|
409
409
|
}
|
|
410
410
|
|
|
411
411
|
await managed.start()
|
|
412
412
|
await managed.stop()
|
|
413
413
|
|
|
414
414
|
// The graceful stop uses the configured signal (a SIGKILL fallback, if any, comes after).
|
|
415
|
-
assert.
|
|
415
|
+
assert.deepEqual(signals, ["SIGINT", "SIGINT"])
|
|
416
416
|
assert.equal(managed.status().state, "stopped")
|
|
417
417
|
})
|
|
418
418
|
|
|
@@ -431,11 +431,11 @@ test("indefinite stop waits for the process to exit without SIGKILL", async () =
|
|
|
431
431
|
})
|
|
432
432
|
/** @type {string[]} */
|
|
433
433
|
const signals = []
|
|
434
|
-
const
|
|
434
|
+
const killProcess = managed.killProcess.bind(managed)
|
|
435
435
|
|
|
436
|
-
managed.
|
|
436
|
+
managed.killProcess = (pid, signal) => {
|
|
437
437
|
signals.push(signal)
|
|
438
|
-
|
|
438
|
+
killProcess(pid, signal)
|
|
439
439
|
}
|
|
440
440
|
|
|
441
441
|
try {
|
|
@@ -443,7 +443,7 @@ test("indefinite stop waits for the process to exit without SIGKILL", async () =
|
|
|
443
443
|
await managed.stop()
|
|
444
444
|
|
|
445
445
|
assert.equal(managed.status().state, "stopped")
|
|
446
|
-
assert.deepEqual(signals, ["SIGTERM"])
|
|
446
|
+
assert.deepEqual(signals, ["SIGTERM", "SIGTERM"])
|
|
447
447
|
} finally {
|
|
448
448
|
await managed.stop()
|
|
449
449
|
}
|
|
@@ -491,6 +491,42 @@ test("stop waits for process group descendants after the detached shell exits",
|
|
|
491
491
|
}
|
|
492
492
|
})
|
|
493
493
|
|
|
494
|
+
test("stop does not return while a gracefully stopped descendant remains unreaped", async () => {
|
|
495
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-process-group-"))
|
|
496
|
+
const pidPath = path.join(dir, "child.pid")
|
|
497
|
+
const child = [
|
|
498
|
+
"const fs = require('node:fs')",
|
|
499
|
+
`fs.writeFileSync(${JSON.stringify(pidPath)}, String(process.pid))`,
|
|
500
|
+
"process.on('SIGTERM', () => setTimeout(() => process.exit(0), 50))",
|
|
501
|
+
"setInterval(() => {}, 1000)"
|
|
502
|
+
].join("; ")
|
|
503
|
+
const managed = new ManagedProcess({
|
|
504
|
+
command: `trap 'exit 0' TERM; ${JSON.stringify(process.execPath)} -e ${JSON.stringify(child)} & wait`,
|
|
505
|
+
cwd: undefined,
|
|
506
|
+
env: {},
|
|
507
|
+
id: "worker",
|
|
508
|
+
logger: () => {},
|
|
509
|
+
outputLines: 50,
|
|
510
|
+
restartDelayMs: 10,
|
|
511
|
+
shouldRestart: () => false,
|
|
512
|
+
stopSignal: "SIGTERM",
|
|
513
|
+
stopTimeoutMs: 2000
|
|
514
|
+
})
|
|
515
|
+
|
|
516
|
+
try {
|
|
517
|
+
await managed.start()
|
|
518
|
+
await waitFor(() => fs.existsSync(pidPath))
|
|
519
|
+
const childPid = Number(fs.readFileSync(pidPath, "utf8"))
|
|
520
|
+
|
|
521
|
+
await managed.stop()
|
|
522
|
+
|
|
523
|
+
assert.throws(() => process.kill(childPid, 0), {code: "ESRCH"})
|
|
524
|
+
} finally {
|
|
525
|
+
await managed.stop()
|
|
526
|
+
fs.rmSync(dir, {force: true, recursive: true})
|
|
527
|
+
}
|
|
528
|
+
})
|
|
529
|
+
|
|
494
530
|
test("a memory restart respawns and is counted when the supervisor still wants the process", async () => {
|
|
495
531
|
const managed = buildLongLived(() => true)
|
|
496
532
|
|