rollbridge 0.1.19 → 0.1.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/docs/cli.md +8 -1
- package/package.json +1 -1
- package/src/cli.js +38 -5
- package/src/daemon.js +68 -6
- package/src/managed-process.js +80 -24
- package/src/release-group.js +5 -0
- package/test/daemon-bootstrap.test.js +229 -18
- package/test/fixtures/owned-child.js +19 -0
- package/test/managed-process.test.js +44 -8
- package/test/rollbridge.test.js +8 -0
- package/test/shutdown-completion.test.js +37 -0
package/README.md
CHANGED
|
@@ -412,6 +412,15 @@ rollbridge daemon --config /srv/ticket-server/rollbridge.js \
|
|
|
412
412
|
--boot-attestation sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
|
|
413
413
|
```
|
|
414
414
|
|
|
415
|
+
External supervisors that need to replace a foreground owner without waiting
|
|
416
|
+
for its workers to drain add `--takeover-owner`. The candidate starts and
|
|
417
|
+
health-checks the exact release first. Only then does it retire the accepted
|
|
418
|
+
owner's proxy/control listeners; the retired daemon keeps its already-accepted
|
|
419
|
+
workers until their ordinary drain completes while the attested replacement
|
|
420
|
+
binds the stable listeners. Candidate bootstrap failure leaves the accepted
|
|
421
|
+
owner untouched. This is opt-in; ordinary daemon bootstrap and `shutdown` keep
|
|
422
|
+
their existing behavior.
|
|
423
|
+
|
|
415
424
|
The four bootstrap inputs are all-or-nothing and use absolute config/release
|
|
416
425
|
paths. Rollbridge binds its proxy, activates the release through the normal
|
|
417
426
|
deploy path, then exposes the control socket and stays foreground. A failed
|
package/docs/cli.md
CHANGED
|
@@ -23,7 +23,7 @@ process-policy details.
|
|
|
23
23
|
## `daemon`
|
|
24
24
|
|
|
25
25
|
```
|
|
26
|
-
rollbridge daemon [--config <path>]
|
|
26
|
+
rollbridge daemon [--config <path>] [--takeover-owner]
|
|
27
27
|
[--release-path <path> --release-id <id> --revision <sha>
|
|
28
28
|
[--boot-attestation <sha256:digest>]]
|
|
29
29
|
```
|
|
@@ -64,6 +64,13 @@ identity. Rollbridge does not calculate or interpret the digest.
|
|
|
64
64
|
With no release options, daemon behavior is unchanged: it starts listener-only
|
|
65
65
|
and waits for control-socket deployments.
|
|
66
66
|
|
|
67
|
+
`--takeover-owner` requires the complete bootstrap tuple. It bootstraps and
|
|
68
|
+
health-checks the replacement before sending the current daemon the private
|
|
69
|
+
retirement command. Retirement stops stable listeners and new work promptly,
|
|
70
|
+
but does not wait for owned workers' normal drain before the replacement binds
|
|
71
|
+
the proxy and control socket. A bootstrap failure occurs before retirement, so
|
|
72
|
+
the previously accepted owner remains available.
|
|
73
|
+
|
|
67
74
|
## `ensure-daemon`
|
|
68
75
|
|
|
69
76
|
```
|
package/package.json
CHANGED
package/src/cli.js
CHANGED
|
@@ -36,6 +36,7 @@ export async function runCli(argv) {
|
|
|
36
36
|
.option("--release-id <id>", "Bootstrap release id (requires --config, --release-path, and --revision)")
|
|
37
37
|
.option("--revision <sha>", "Bootstrap revision (requires --config, --release-path, and --release-id)")
|
|
38
38
|
.option("--boot-attestation <digest>", "Opaque bootstrap ownership attestation (requires the complete bootstrap release tuple)")
|
|
39
|
+
.option("--takeover-owner", "Boot and health-check before retiring the current external owner")
|
|
39
40
|
.action(async (options) => {
|
|
40
41
|
const bootstrap = await validateDaemonBootstrapOptions(options)
|
|
41
42
|
const configPath = await resolveConfigPath(options.config)
|
|
@@ -43,7 +44,9 @@ export async function runCli(argv) {
|
|
|
43
44
|
const runtime = await loadDaemonRuntimeIdentity(process.env.ROLLBRIDGE_DAEMON_RUNTIME_MANIFEST)
|
|
44
45
|
const daemon = new RollbridgeDaemon({bootstrap, config, configPath, runtime})
|
|
45
46
|
|
|
46
|
-
|
|
47
|
+
if (options.takeoverOwner && (!bootstrap || !bootstrap.attestation)) throw new Error("Daemon --takeover-owner requires the complete bootstrap release tuple and --boot-attestation.")
|
|
48
|
+
|
|
49
|
+
if (!options.takeoverOwner) await daemon.start({exposeControl: !bootstrap})
|
|
47
50
|
|
|
48
51
|
const shutdown = async () => {
|
|
49
52
|
await daemon.shutdown()
|
|
@@ -56,10 +59,25 @@ export async function runCli(argv) {
|
|
|
56
59
|
if (bootstrap) {
|
|
57
60
|
try {
|
|
58
61
|
await daemon.deploy(bootstrap)
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
62
|
+
if (options.takeoverOwner) {
|
|
63
|
+
await sendControlCommand({command: {attestation: bootstrap.attestation, command: "retire-owner"}, path: config.control.path})
|
|
64
|
+
await daemon.start({reportOrphans: false})
|
|
65
|
+
} else {
|
|
66
|
+
await daemon.exposeControl()
|
|
67
|
+
}
|
|
68
|
+
} catch (error) {
|
|
69
|
+
const failure = error instanceof Error ? error : String(error)
|
|
70
|
+
|
|
71
|
+
daemon.logger("bootstrap activation failed", {releaseId: bootstrap.releaseId, status: "error", ...errorLogData(failure)})
|
|
72
|
+
|
|
73
|
+
try {
|
|
74
|
+
await daemon.shutdown()
|
|
75
|
+
} catch (shutdownError) {
|
|
76
|
+
const shutdownFailure = shutdownError instanceof Error ? shutdownError : String(shutdownError)
|
|
77
|
+
|
|
78
|
+
daemon.logger("bootstrap shutdown failed", {releaseId: bootstrap.releaseId, status: "error", ...errorLogData(shutdownFailure)})
|
|
79
|
+
}
|
|
80
|
+
|
|
63
81
|
process.exitCode = 1
|
|
64
82
|
return
|
|
65
83
|
}
|
|
@@ -952,3 +970,18 @@ function isMissingDaemonError(error) {
|
|
|
952
970
|
|
|
953
971
|
return error.code === "ENOENT" || error.code === "ECONNREFUSED"
|
|
954
972
|
}
|
|
973
|
+
|
|
974
|
+
/**
|
|
975
|
+
* Converts any thrown value into JSON-safe diagnostics without losing an Error stack.
|
|
976
|
+
* @param {Error | string} error - Thrown value.
|
|
977
|
+
* @returns {{error: string, stack: string}} Safe structured log fields.
|
|
978
|
+
*/
|
|
979
|
+
function errorLogData(error) {
|
|
980
|
+
if (error instanceof Error) {
|
|
981
|
+
return {error: error.message, stack: error.stack || `${error.name}: ${error.message}`}
|
|
982
|
+
}
|
|
983
|
+
|
|
984
|
+
const message = String(error)
|
|
985
|
+
|
|
986
|
+
return {error: message, stack: message}
|
|
987
|
+
}
|
package/src/daemon.js
CHANGED
|
@@ -56,13 +56,17 @@ export default class RollbridgeDaemon {
|
|
|
56
56
|
this.proxy = httpProxy.createProxyServer({ws: true, xfwd: true})
|
|
57
57
|
this.proxyServer = /** @type {http.Server | undefined} */ (undefined)
|
|
58
58
|
this.controlServer = /** @type {net.Server | undefined} */ (undefined)
|
|
59
|
+
this.controlSocketOwned = false
|
|
59
60
|
this.controlSockets = /** @type {Set<net.Socket>} */ (new Set())
|
|
60
61
|
this.proxyPort = /** @type {number | undefined} */ (undefined)
|
|
61
62
|
this.stopping = false
|
|
62
63
|
this.statePath = config.statePath
|
|
63
64
|
this.persistTimer = /** @type {ReturnType<typeof setInterval> | undefined} */ (undefined)
|
|
65
|
+
this.persistenceEnabled = false
|
|
64
66
|
this.pendingWrite = /** @type {Promise<void> | undefined} */ (undefined)
|
|
67
|
+
this.stateCleanupEnabled = false
|
|
65
68
|
this.shutdownPromise = /** @type {Promise<void> | undefined} */ (undefined)
|
|
69
|
+
this.retirementPromise = /** @type {Promise<void> | undefined} */ (undefined)
|
|
66
70
|
this.controlClosePromise = /** @type {Promise<void> | undefined} */ (undefined)
|
|
67
71
|
this.startingReleases = /** @type {Set<ReleaseGroup>} */ (new Set())
|
|
68
72
|
// Still-alive managed processes left by a previous daemon (from statePath), captured at
|
|
@@ -74,11 +78,11 @@ export default class RollbridgeDaemon {
|
|
|
74
78
|
|
|
75
79
|
/**
|
|
76
80
|
* Starts daemon listeners.
|
|
77
|
-
* @param {{exposeControl?: boolean}} [options] -
|
|
81
|
+
* @param {{exposeControl?: boolean, reportOrphans?: boolean}} [options] - Listener startup options.
|
|
78
82
|
* @returns {Promise<void>} Resolves when the requested listeners are ready.
|
|
79
83
|
*/
|
|
80
|
-
async start({exposeControl = true} = {}) {
|
|
81
|
-
await this.reportOrphans()
|
|
84
|
+
async start({exposeControl = true, reportOrphans = true} = {}) {
|
|
85
|
+
if (reportOrphans) await this.reportOrphans()
|
|
82
86
|
await this.startProxy()
|
|
83
87
|
if (exposeControl) await this.exposeControl()
|
|
84
88
|
}
|
|
@@ -126,6 +130,7 @@ export default class RollbridgeDaemon {
|
|
|
126
130
|
await new Promise((resolve, reject) => {
|
|
127
131
|
server.once("error", reject)
|
|
128
132
|
server.listen(this.config.control.path, () => {
|
|
133
|
+
this.controlSocketOwned = true
|
|
129
134
|
this.logger("control socket listening", {path: this.config.control.path})
|
|
130
135
|
resolve(undefined)
|
|
131
136
|
})
|
|
@@ -358,6 +363,13 @@ export default class RollbridgeDaemon {
|
|
|
358
363
|
return {message: "shutdown"}
|
|
359
364
|
}
|
|
360
365
|
|
|
366
|
+
if (commandName === "retire-owner") {
|
|
367
|
+
const attestation = requiredString(data.attestation, "attestation")
|
|
368
|
+
if (!/^sha256:[a-f0-9]{64}$/.test(attestation)) throw new Error("Owner retirement attestation must use the canonical sha256:<64 lowercase hex> format")
|
|
369
|
+
await this.retireOwner({attestation, completionSocket: controlSocket})
|
|
370
|
+
return {message: "owner retired"}
|
|
371
|
+
}
|
|
372
|
+
|
|
361
373
|
throw new Error(`Unknown command: ${String(commandName)}`)
|
|
362
374
|
}
|
|
363
375
|
|
|
@@ -732,6 +744,8 @@ export default class RollbridgeDaemon {
|
|
|
732
744
|
startStatePersistence() {
|
|
733
745
|
if (!this.statePath) return
|
|
734
746
|
|
|
747
|
+
this.stateCleanupEnabled = true
|
|
748
|
+
this.persistenceEnabled = true
|
|
735
749
|
this.persistState()
|
|
736
750
|
this.persistTimer = setInterval(() => this.persistState(), STATE_PERSIST_INTERVAL_MS)
|
|
737
751
|
this.persistTimer.unref?.()
|
|
@@ -743,7 +757,7 @@ export default class RollbridgeDaemon {
|
|
|
743
757
|
* @returns {void}
|
|
744
758
|
*/
|
|
745
759
|
persistState() {
|
|
746
|
-
if (!this.statePath || this.stopping) return
|
|
760
|
+
if (!this.statePath || !this.persistenceEnabled || this.stopping) return
|
|
747
761
|
|
|
748
762
|
const statePath = this.statePath
|
|
749
763
|
const status = /** @type {Record<string, JsonValue>} */ (secretSafeStateValue(this.status()))
|
|
@@ -770,6 +784,7 @@ export default class RollbridgeDaemon {
|
|
|
770
784
|
async reportOrphans() {
|
|
771
785
|
if (!this.statePath) return
|
|
772
786
|
|
|
787
|
+
this.stateCleanupEnabled = true
|
|
773
788
|
const orphans = liveProcesses(await readState(this.statePath))
|
|
774
789
|
|
|
775
790
|
// Keep them for status() so `rollbridge status` reflects still-running children after a
|
|
@@ -797,6 +812,50 @@ export default class RollbridgeDaemon {
|
|
|
797
812
|
if (waitForControlConnections && this.controlClosePromise) await this.controlClosePromise
|
|
798
813
|
}
|
|
799
814
|
|
|
815
|
+
/**
|
|
816
|
+
* Relinquishes stable listeners promptly while retaining draining children under
|
|
817
|
+
* this daemon until their normal stop contract completes.
|
|
818
|
+
* @param {{attestation: string, completionSocket?: net.Socket}} options - Attested handoff request.
|
|
819
|
+
* @returns {Promise<void>} Resolves once a replacement can exclusively bind listeners.
|
|
820
|
+
*/
|
|
821
|
+
async retireOwner({attestation, completionSocket}) {
|
|
822
|
+
if (this.retirementPromise) return await this.retirementPromise
|
|
823
|
+
this.retirementPromise = this.performOwnerRetirement(attestation, completionSocket)
|
|
824
|
+
return await this.retirementPromise
|
|
825
|
+
}
|
|
826
|
+
|
|
827
|
+
/**
|
|
828
|
+
* @param {string} attestation - Replacement boot attestation.
|
|
829
|
+
* @param {net.Socket | undefined} completionSocket - Requesting handoff connection.
|
|
830
|
+
* @returns {Promise<void>} Resolves after quiesce and listener release.
|
|
831
|
+
*/
|
|
832
|
+
async performOwnerRetirement(attestation, completionSocket) {
|
|
833
|
+
this.stopping = true
|
|
834
|
+
if (this.persistTimer) {
|
|
835
|
+
clearInterval(this.persistTimer)
|
|
836
|
+
this.persistTimer = undefined
|
|
837
|
+
}
|
|
838
|
+
this.persistenceEnabled = false
|
|
839
|
+
if (this.pendingWrite) await this.pendingWrite
|
|
840
|
+
this.controlClosePromise = this.closeServer(this.controlServer)
|
|
841
|
+
for (const socket of this.controlSockets) if (socket !== completionSocket) socket.destroy()
|
|
842
|
+
await Promise.all([
|
|
843
|
+
...[...this.services.values()].map((processInstance) => processInstance.quiesce()),
|
|
844
|
+
...[...this.singletons.values()].map((processInstance) => processInstance.quiesce()),
|
|
845
|
+
...[...this.startingReleases].map((release) => release.quiesce()),
|
|
846
|
+
...[...this.releases.values()].map((release) => release.quiesce())
|
|
847
|
+
])
|
|
848
|
+
await this.removeControlSocket()
|
|
849
|
+
void this.closeServer(this.proxyServer)
|
|
850
|
+
void Promise.allSettled([
|
|
851
|
+
...[...this.services.values()].map((processInstance) => processInstance.stop()),
|
|
852
|
+
...[...this.singletons.values()].map((processInstance) => processInstance.stop()),
|
|
853
|
+
...[...this.startingReleases].map((release) => release.stop()),
|
|
854
|
+
...[...this.releases.values()].map((release) => release.stop())
|
|
855
|
+
])
|
|
856
|
+
this.logger("external owner retired", {attestation, status: "draining"})
|
|
857
|
+
}
|
|
858
|
+
|
|
800
859
|
/**
|
|
801
860
|
* @param {net.Socket | undefined} completionSocket - Requester retained for the final response.
|
|
802
861
|
* @returns {Promise<void>} Retires listeners and cleans up every daemon-owned resource.
|
|
@@ -834,7 +893,7 @@ export default class RollbridgeDaemon {
|
|
|
834
893
|
// new writes start: stopping is set and the persist timer is cleared above). Prior-daemon
|
|
835
894
|
// orphans are not owned by this daemon, so retain their records until they are confirmed gone.
|
|
836
895
|
await captureShutdownError(cleanupErrors, "persistent state cleanup", async () => {
|
|
837
|
-
if (!this.statePath) return
|
|
896
|
+
if (!this.statePath || !this.stateCleanupEnabled) return
|
|
838
897
|
if (this.pendingWrite) await this.pendingWrite
|
|
839
898
|
const orphans = this.orphans.filter((orphan) => isProcessAlive(orphan.pid))
|
|
840
899
|
|
|
@@ -856,7 +915,10 @@ export default class RollbridgeDaemon {
|
|
|
856
915
|
|
|
857
916
|
/** @returns {Promise<void>} Removes the configured control socket path. */
|
|
858
917
|
async removeControlSocket() {
|
|
918
|
+
if (!this.controlSocketOwned) return
|
|
919
|
+
|
|
859
920
|
await fs.rm(this.config.control.path, {force: true})
|
|
921
|
+
this.controlSocketOwned = false
|
|
860
922
|
}
|
|
861
923
|
|
|
862
924
|
/**
|
|
@@ -925,7 +987,7 @@ function isShutdownControlLine(line) {
|
|
|
925
987
|
try {
|
|
926
988
|
const command = JSON.parse(line)
|
|
927
989
|
|
|
928
|
-
return Boolean(command && typeof command === "object" &&
|
|
990
|
+
return Boolean(command && typeof command === "object" && ["retire-owner", "shutdown"].includes(command.command))
|
|
929
991
|
} catch {
|
|
930
992
|
return false
|
|
931
993
|
}
|
package/src/managed-process.js
CHANGED
|
@@ -62,6 +62,8 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
62
62
|
this.memoryWarned = false
|
|
63
63
|
this.startedAtMs = /** @type {number | undefined} */ (undefined)
|
|
64
64
|
this.intentionalStop = false
|
|
65
|
+
this.intentionalStopSignal = /** @type {ProcessExitSignal | undefined} */ (undefined)
|
|
66
|
+
this.quiescePromise = /** @type {Promise<void> | undefined} */ (undefined)
|
|
65
67
|
this.restartTimer = undefined
|
|
66
68
|
this.child = undefined
|
|
67
69
|
this.exitPromise = undefined
|
|
@@ -78,6 +80,8 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
78
80
|
if (this.child) return
|
|
79
81
|
|
|
80
82
|
this.intentionalStop = false
|
|
83
|
+
this.intentionalStopSignal = undefined
|
|
84
|
+
this.quiescePromise = undefined
|
|
81
85
|
this.exitCode = undefined
|
|
82
86
|
this.exitSignal = undefined
|
|
83
87
|
this.state = "starting"
|
|
@@ -165,8 +169,8 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
165
169
|
onExit(code, signal) {
|
|
166
170
|
const wasIntentional = this.intentionalStop
|
|
167
171
|
|
|
168
|
-
this.exitCode = code
|
|
169
|
-
this.exitSignal = signal
|
|
172
|
+
this.exitCode = this.intentionalStopSignal ? null : code
|
|
173
|
+
this.exitSignal = signal ?? this.intentionalStopSignal
|
|
170
174
|
this.child = undefined
|
|
171
175
|
this.pid = undefined
|
|
172
176
|
this.exitPromise = undefined
|
|
@@ -338,36 +342,22 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
338
342
|
* @returns {Promise<void>} Resolves when stopped.
|
|
339
343
|
*/
|
|
340
344
|
async stop(options = {}) {
|
|
341
|
-
this.
|
|
342
|
-
this.
|
|
343
|
-
|
|
344
|
-
if (this.restartTimer) {
|
|
345
|
-
clearTimeout(this.restartTimer)
|
|
346
|
-
this.restartTimer = undefined
|
|
347
|
-
}
|
|
348
|
-
|
|
349
|
-
const child = this.child
|
|
345
|
+
const pgid = this.child?.pid ?? this.pid
|
|
346
|
+
const exitPromise = this.exitPromise
|
|
347
|
+
await this.quiesce()
|
|
350
348
|
|
|
351
|
-
if (!
|
|
349
|
+
if (!pgid) {
|
|
352
350
|
this.state = "stopped"
|
|
353
351
|
return
|
|
354
352
|
}
|
|
355
353
|
|
|
356
|
-
const
|
|
357
|
-
const exitPromise = this.exitPromise
|
|
358
|
-
|
|
359
|
-
this.state = "stopping"
|
|
360
|
-
|
|
361
|
-
const {drainCommand, drainTimeoutMs, quietCommand, stopCommand} = this.lifecycle
|
|
354
|
+
const {drainCommand, drainTimeoutMs, stopCommand} = this.lifecycle
|
|
362
355
|
|
|
363
356
|
const hookTimeoutMs = this.hookTimeoutMs()
|
|
364
357
|
|
|
365
|
-
// 1. Quiesce: tell the process to stop accepting new work.
|
|
366
|
-
if (quietCommand) await this.runHook(quietCommand, hookTimeoutMs, "quiet command")
|
|
367
|
-
|
|
368
358
|
// 2. Drain: let in-flight work finish, bounded by drainTimeoutMs (0 skips the step). A
|
|
369
359
|
// drainCommand blocks until drained; otherwise wait for the process to exit on its own.
|
|
370
|
-
if (this.
|
|
360
|
+
if (this.processGroupExists(pgid) && drainTimeoutMs > 0) {
|
|
371
361
|
if (drainCommand) await this.runHook(drainCommand, drainTimeoutMs, "drain command")
|
|
372
362
|
else await this.waitForExit(drainTimeoutMs)
|
|
373
363
|
}
|
|
@@ -375,13 +365,16 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
375
365
|
// 3. Stop whatever is still running, then SIGKILL if it outlasts the graceful window.
|
|
376
366
|
if (this.processGroupExists(pgid)) {
|
|
377
367
|
if (stopCommand) await this.runHook(stopCommand, hookTimeoutMs, "stop command", pgid)
|
|
378
|
-
else
|
|
368
|
+
else {
|
|
369
|
+
this.intentionalStopSignal = /** @type {ProcessExitSignal} */ (this.stopSignal)
|
|
370
|
+
await this.signalProcessGroup(this.stopSignal, pgid, options.timeoutMs ?? this.stopTimeoutMs)
|
|
371
|
+
}
|
|
379
372
|
|
|
380
373
|
const timeoutMs = options.timeoutMs ?? this.stopTimeoutMs
|
|
381
374
|
|
|
382
375
|
if (!(await this.waitForProcessGroupExit(pgid, timeoutMs))) {
|
|
383
376
|
this.logger("process stop timed out; sending SIGKILL", {id: this.id, pid: pgid})
|
|
384
|
-
this.
|
|
377
|
+
await this.signalProcessGroup("SIGKILL", pgid, 5000)
|
|
385
378
|
await this.waitForProcessGroupExit(pgid, 5000)
|
|
386
379
|
}
|
|
387
380
|
}
|
|
@@ -391,6 +384,26 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
391
384
|
this.state = "stopped"
|
|
392
385
|
}
|
|
393
386
|
|
|
387
|
+
/** @returns {Promise<void>} Stops restarts and waits until the process has stopped accepting new work. */
|
|
388
|
+
async quiesce() {
|
|
389
|
+
if (this.quiescePromise) return await this.quiescePromise
|
|
390
|
+
this.quiescePromise = (async () => {
|
|
391
|
+
this.intentionalStop = true
|
|
392
|
+
this.clearMemoryMonitor()
|
|
393
|
+
if (this.restartTimer) {
|
|
394
|
+
clearTimeout(this.restartTimer)
|
|
395
|
+
this.restartTimer = undefined
|
|
396
|
+
}
|
|
397
|
+
if (!this.child?.pid) {
|
|
398
|
+
this.state = "stopped"
|
|
399
|
+
return
|
|
400
|
+
}
|
|
401
|
+
this.state = "stopping"
|
|
402
|
+
if (this.lifecycle.quietCommand) await this.runHook(this.lifecycle.quietCommand, this.hookTimeoutMs(), "quiet command")
|
|
403
|
+
})()
|
|
404
|
+
return await this.quiescePromise
|
|
405
|
+
}
|
|
406
|
+
|
|
394
407
|
/** @returns {number} Timeout used for lifecycle hooks. */
|
|
395
408
|
hookTimeoutMs() {
|
|
396
409
|
if (this.stopTimeoutMs === "indefinite") return 30000
|
|
@@ -481,6 +494,49 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
481
494
|
}
|
|
482
495
|
}
|
|
483
496
|
|
|
497
|
+
/**
|
|
498
|
+
* Signals descendants before their shell leader so the leader can reap them before it exits.
|
|
499
|
+
* Falls back to the portable group signal when procfs cannot identify group members.
|
|
500
|
+
* @param {string} signal - Signal name.
|
|
501
|
+
* @param {number} pgid - Process group id.
|
|
502
|
+
* @param {StopTimeoutMs} timeoutMs - Maximum time to wait for descendant reaping.
|
|
503
|
+
* @returns {Promise<void>} Resolves after the leader has been signalled.
|
|
504
|
+
*/
|
|
505
|
+
async signalProcessGroup(signal, pgid, timeoutMs) {
|
|
506
|
+
const members = processGroupMembers(pgid)
|
|
507
|
+
const descendants = members.filter((member) => member.pid !== pgid)
|
|
508
|
+
|
|
509
|
+
if (descendants.length === 0) {
|
|
510
|
+
this.killProcessGroup(signal, pgid)
|
|
511
|
+
return
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
for (const descendant of descendants) this.killProcess(descendant.pid, signal)
|
|
515
|
+
const deadline = timeoutMs === "indefinite" ? undefined : Date.now() + timeoutMs
|
|
516
|
+
|
|
517
|
+
while (processGroupMembers(pgid).some((member) => member.pid !== pgid)) {
|
|
518
|
+
if (deadline !== undefined && Date.now() >= deadline) break
|
|
519
|
+
await new Promise((resolve) => setTimeout(resolve, 25))
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
this.killProcess(pgid, signal)
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
/**
|
|
526
|
+
* Signals one verified member of the owned process group.
|
|
527
|
+
* @param {number} pid - Process id.
|
|
528
|
+
* @param {string} signal - Signal name.
|
|
529
|
+
* @returns {void}
|
|
530
|
+
*/
|
|
531
|
+
killProcess(pid, signal) {
|
|
532
|
+
try {
|
|
533
|
+
process.kill(pid, signal)
|
|
534
|
+
} catch (error) {
|
|
535
|
+
if (error && typeof error === "object" && "code" in error && error.code === "ESRCH") return
|
|
536
|
+
throw error
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
|
|
484
540
|
/**
|
|
485
541
|
* @param {number} pgid - Process group id.
|
|
486
542
|
* @returns {boolean} True until the process group no longer exists.
|
package/src/release-group.js
CHANGED
|
@@ -403,6 +403,11 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
403
403
|
this.stoppedAt = new Date().toISOString()
|
|
404
404
|
}
|
|
405
405
|
|
|
406
|
+
/** @returns {Promise<void>} Quiesces every release process without waiting for its drain. */
|
|
407
|
+
async quiesce() {
|
|
408
|
+
await Promise.all([...this.processes.values()].map((processInstance) => processInstance.quiesce()))
|
|
409
|
+
}
|
|
410
|
+
|
|
406
411
|
/** @returns {ReleaseStatus} Status payload. */
|
|
407
412
|
status() {
|
|
408
413
|
return {
|
|
@@ -4,6 +4,7 @@ import assert from "node:assert/strict"
|
|
|
4
4
|
import {spawn} from "node:child_process"
|
|
5
5
|
import {once} from "node:events"
|
|
6
6
|
import fs from "node:fs/promises"
|
|
7
|
+
import net from "node:net"
|
|
7
8
|
import os from "node:os"
|
|
8
9
|
import path from "node:path"
|
|
9
10
|
import test from "node:test"
|
|
@@ -14,9 +15,12 @@ import {isProcessAlive, liveProcesses, readState, writeState} from "../src/state
|
|
|
14
15
|
const currentDir = path.dirname(fileURLToPath(import.meta.url))
|
|
15
16
|
const binPath = path.join(currentDir, "..", "bin", "rollbridge")
|
|
16
17
|
const dummyAppPath = path.join(currentDir, "fixtures", "dummy-app.js")
|
|
18
|
+
const ownedChildPath = path.join(currentDir, "fixtures", "owned-child.js")
|
|
17
19
|
const firstAttestation = `sha256:${"a".repeat(64)}`
|
|
18
20
|
const secondAttestation = `sha256:${"b".repeat(64)}`
|
|
19
21
|
|
|
22
|
+
/** @typedef {{data?: Record<string, import("../src/json.js").JsonValue>, message?: string}} StructuredRecord */
|
|
23
|
+
|
|
20
24
|
test("daemon bootstrap requires complete, safe, absolute inputs before binding listeners", async (t) => {
|
|
21
25
|
const cases = [
|
|
22
26
|
{args: ["--config", "CONFIG", "--release-path", "RELEASE", "--release-id", "v1"], message: /must be provided together/},
|
|
@@ -85,6 +89,49 @@ test("daemon bootstrap activates the exact release through the foreground daemon
|
|
|
85
89
|
}
|
|
86
90
|
})
|
|
87
91
|
|
|
92
|
+
test("failed takeover bootstrap preserves the previously accepted owner", async () => {
|
|
93
|
+
const fixture = await createFixture({persistState: true})
|
|
94
|
+
const accepted = spawnDaemon(fixture, {attestation: firstAttestation, releaseId: "accepted", revision: "accepted123"})
|
|
95
|
+
|
|
96
|
+
try {
|
|
97
|
+
await waitForLog(accepted, "control socket listening")
|
|
98
|
+
await waitForFile(fixture.statePath)
|
|
99
|
+
const badConfig = JSON.parse((await fs.readFile(fixture.configPath, "utf8")).replace(/^module\.exports = /, ""))
|
|
100
|
+
badConfig.processes[0].health.path = "/never-ready"
|
|
101
|
+
badConfig.processes[0].health.timeoutMs = 100
|
|
102
|
+
await fs.writeFile(fixture.configPath, `module.exports = ${JSON.stringify(badConfig, null, 2)}\n`)
|
|
103
|
+
|
|
104
|
+
const result = await runDaemon([
|
|
105
|
+
"--config", fixture.configPath,
|
|
106
|
+
"--release-path", fixture.root,
|
|
107
|
+
"--release-id", "candidate",
|
|
108
|
+
"--revision", "candidate123",
|
|
109
|
+
"--boot-attestation", secondAttestation,
|
|
110
|
+
"--takeover-owner"
|
|
111
|
+
])
|
|
112
|
+
|
|
113
|
+
assert.notEqual(result.code, 0)
|
|
114
|
+
const records = parseStructuredOutput(result.output)
|
|
115
|
+
const failure = records.find((record) => record.message === "bootstrap activation failed")
|
|
116
|
+
const candidatePid = Number(await fs.readFile(fixture.startedPath, "utf8"))
|
|
117
|
+
|
|
118
|
+
assert.match(String(failure?.data?.error), /Health check failed/)
|
|
119
|
+
assert.match(String(failure?.data?.stack), /Error: Health check failed/)
|
|
120
|
+
assert.equal(isProcessAlive(candidatePid), false, "the failed candidate process must be stopped")
|
|
121
|
+
const status = await sendControlCommand({command: {command: "status"}, path: fixture.socketPath})
|
|
122
|
+
assert.equal(status.activeReleaseId, "accepted")
|
|
123
|
+
assert.ok(status.bootstrap && typeof status.bootstrap === "object" && !Array.isArray(status.bootstrap))
|
|
124
|
+
assert.equal(status.bootstrap.attestation, firstAttestation)
|
|
125
|
+
const priorState = await readState(fixture.statePath)
|
|
126
|
+
|
|
127
|
+
assert.ok(priorState && typeof priorState === "object" && !Array.isArray(priorState) && priorState.activeReleaseId === "accepted", "candidate cleanup must preserve the prior owner's state")
|
|
128
|
+
} finally {
|
|
129
|
+
accepted.kill("SIGTERM")
|
|
130
|
+
if (accepted.exitCode === null) await once(accepted, "exit")
|
|
131
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
132
|
+
}
|
|
133
|
+
})
|
|
134
|
+
|
|
88
135
|
test("daemon bootstrap does not expose control deploys until activation completes", async () => {
|
|
89
136
|
const fixture = await createFixture({healthGate: true, healthTimeoutMs: 60000})
|
|
90
137
|
const started = waitForFile(fixture.startedPath)
|
|
@@ -235,24 +282,56 @@ test("SIGTERM during multi-process bootstrap owns every process started after sh
|
|
|
235
282
|
}
|
|
236
283
|
})
|
|
237
284
|
|
|
238
|
-
test("failed
|
|
239
|
-
const fixture = await createFixture({
|
|
285
|
+
test("failed ordinary bootstrap completely shuts down attempt-owned resources and exposes the cause", async () => {
|
|
286
|
+
const fixture = await createFixture({attemptOwnedProcesses: true, fixedPorts: true, missingControlParent: true})
|
|
240
287
|
|
|
241
288
|
try {
|
|
242
289
|
const result = await runDaemon([
|
|
243
290
|
"--config", fixture.configPath,
|
|
244
291
|
"--release-path", fixture.root,
|
|
245
|
-
"--release-id", "
|
|
246
|
-
"--revision", "
|
|
292
|
+
"--release-id", "ordinary-failure",
|
|
293
|
+
"--revision", "ordinary123"
|
|
247
294
|
])
|
|
248
|
-
const records = result.output
|
|
295
|
+
const records = parseStructuredOutput(result.output)
|
|
249
296
|
const failure = records.find((record) => record.message === "bootstrap activation failed")
|
|
250
297
|
|
|
251
298
|
assert.notEqual(result.code, 0)
|
|
252
|
-
assert.
|
|
253
|
-
assert.
|
|
299
|
+
assert.equal(failure?.data?.releaseId, "ordinary-failure")
|
|
300
|
+
assert.equal(failure?.data?.status, "error")
|
|
301
|
+
assert.match(String(failure?.data?.error), /listen (?:EACCES|ENOENT)/)
|
|
302
|
+
assert.match(String(failure?.data?.stack), /Error: listen (?:EACCES|ENOENT)/)
|
|
303
|
+
await assertAttemptResourcesStopped(fixture, records)
|
|
254
304
|
await assert.rejects(() => fs.stat(fixture.socketPath), {code: "ENOENT"})
|
|
255
305
|
} finally {
|
|
306
|
+
await killAttemptProcesses(fixture.lifecyclePath)
|
|
307
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
308
|
+
}
|
|
309
|
+
})
|
|
310
|
+
|
|
311
|
+
test("failed takeover retirement completely shuts down and exits non-zero instead of lingering", async () => {
|
|
312
|
+
const fixture = await createFixture({attemptOwnedProcesses: true, fixedPorts: true})
|
|
313
|
+
|
|
314
|
+
try {
|
|
315
|
+
const result = await runDaemon([
|
|
316
|
+
"--config", fixture.configPath,
|
|
317
|
+
"--release-path", fixture.root,
|
|
318
|
+
"--release-id", "orphaned-candidate",
|
|
319
|
+
"--revision", "candidate123",
|
|
320
|
+
"--boot-attestation", secondAttestation,
|
|
321
|
+
"--takeover-owner"
|
|
322
|
+
], {timeoutMs: 2000})
|
|
323
|
+
const records = parseStructuredOutput(result.output)
|
|
324
|
+
const failure = records.find((record) => record.message === "bootstrap activation failed")
|
|
325
|
+
|
|
326
|
+
assert.notEqual(result.code, 0)
|
|
327
|
+
assert.equal(failure?.data?.releaseId, "orphaned-candidate")
|
|
328
|
+
assert.equal(failure?.data?.status, "error")
|
|
329
|
+
assert.match(String(failure?.data?.error), /connect ENOENT/)
|
|
330
|
+
assert.match(String(failure?.data?.stack), /Error: connect ENOENT/)
|
|
331
|
+
await assertAttemptResourcesStopped(fixture, records)
|
|
332
|
+
await assert.rejects(() => fs.stat(fixture.socketPath), {code: "ENOENT"})
|
|
333
|
+
} finally {
|
|
334
|
+
await killAttemptProcesses(fixture.lifecyclePath)
|
|
256
335
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
257
336
|
}
|
|
258
337
|
})
|
|
@@ -318,12 +397,12 @@ test("failed daemon bootstrap preserves prior live process records in statePath"
|
|
|
318
397
|
})
|
|
319
398
|
|
|
320
399
|
/**
|
|
321
|
-
* @param {{healthGate?: boolean, healthPath?: string, healthTimeoutMs?: number, multiProcessSignal?: boolean, persistState?: boolean}} [options] - Fixture options.
|
|
322
|
-
* @returns {Promise<{configPath: string, gatePath: string, healthGatePath: string, lifecyclePath: string, root: string, socketPath: string, startedPath: string, statePath: string, stoppedPath: string}>} Fixture paths.
|
|
400
|
+
* @param {{attemptOwnedProcesses?: boolean, fixedPorts?: boolean, healthGate?: boolean, healthPath?: string, healthTimeoutMs?: number, missingControlParent?: boolean, multiProcessSignal?: boolean, persistState?: boolean}} [options] - Fixture options.
|
|
401
|
+
* @returns {Promise<{configPath: string, gatePath: string, healthGatePath: string, lifecyclePath: string, processPort: number, proxyPort: number, root: string, socketPath: string, startedPath: string, statePath: string, stoppedPath: string}>} Fixture paths.
|
|
323
402
|
*/
|
|
324
|
-
async function createFixture({healthGate = false, healthPath = "/ping", healthTimeoutMs = 1000, multiProcessSignal = false, persistState = false} = {}) {
|
|
403
|
+
async function createFixture({attemptOwnedProcesses = false, fixedPorts = false, healthGate = false, healthPath = "/ping", healthTimeoutMs = 1000, missingControlParent = false, multiProcessSignal = false, persistState = false} = {}) {
|
|
325
404
|
const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-bootstrap-"))
|
|
326
|
-
const socketPath = path.join(root, "control.sock")
|
|
405
|
+
const socketPath = missingControlParent ? path.join(root, "missing", "control.sock") : path.join(root, "control.sock")
|
|
327
406
|
const statePath = path.join(root, "state.json")
|
|
328
407
|
const startedPath = path.join(root, "started.pid")
|
|
329
408
|
const stoppedPath = path.join(root, "stopped.pid")
|
|
@@ -332,8 +411,12 @@ async function createFixture({healthGate = false, healthPath = "/ping", healthTi
|
|
|
332
411
|
const healthGatePath = path.join(root, "health-ready")
|
|
333
412
|
const configPath = path.join(root, "rollbridge.js")
|
|
334
413
|
const command = `${JSON.stringify(process.execPath)} ${JSON.stringify(dummyAppPath)}`
|
|
414
|
+
const ownedCommand = `exec ${JSON.stringify(process.execPath)} ${JSON.stringify(ownedChildPath)}`
|
|
415
|
+
const ownedWebCommand = `exec ${command}`
|
|
416
|
+
const [processPort, proxyPort] = fixedPorts ? await availablePorts(2) : [0, 0]
|
|
335
417
|
const lifecycleEnv = {ROLLBRIDGE_TEST_LIFECYCLE_PATH: lifecyclePath}
|
|
336
418
|
const webEnv = {
|
|
419
|
+
...(attemptOwnedProcesses ? lifecycleEnv : {}),
|
|
337
420
|
ROLLBRIDGE_TEST_STARTED_PATH: startedPath,
|
|
338
421
|
ROLLBRIDGE_TEST_STOPPED_PATH: stoppedPath,
|
|
339
422
|
...(healthGate ? {ROLLBRIDGE_TEST_HEALTH_GATE_PATH: healthGatePath} : {})
|
|
@@ -345,8 +428,13 @@ async function createFixture({healthGate = false, healthPath = "/ping", healthTi
|
|
|
345
428
|
{command: `trap '' TERM; printf '%s\\n' '{"event":"shutdown"}' >> ${JSON.stringify(lifecyclePath)}; kill -TERM "$ROLLBRIDGE_TEST_DAEMON_PID"; printf '{"event":"started","pid":%s,"processId":"database","replicaIndex":"0"}\\n' "$$" >> ${JSON.stringify(lifecyclePath)}; read ignored < ${JSON.stringify(gatePath)}`, env: lifecycleEnv, id: "database", policy: "service"},
|
|
346
429
|
{command: `exec ${command}`, env: lifecycleEnv, id: "worker", policy: "companion", replicas: 2},
|
|
347
430
|
{command: `exec ${command}`, env: lifecycleEnv, health: {path: healthPath, timeoutMs: healthTimeoutMs}, id: "web", policy: "proxied", port: {from: 0, to: 0}}
|
|
431
|
+
] : attemptOwnedProcesses ? [
|
|
432
|
+
{command: ownedCommand, env: lifecycleEnv, id: "database", policy: "service"},
|
|
433
|
+
{command: ownedCommand, env: lifecycleEnv, id: "worker", policy: "companion"},
|
|
434
|
+
{command: ownedCommand, env: lifecycleEnv, id: "scheduler", policy: "singleton"},
|
|
435
|
+
{command: ownedWebCommand, env: webEnv, health: {path: healthPath, timeoutMs: healthTimeoutMs}, id: "web", policy: "proxied", port: {from: processPort, to: processPort}}
|
|
348
436
|
] : [{command, env: webEnv, health: {path: healthPath, timeoutMs: healthTimeoutMs}, id: "web", policy: "proxied", port: {from: 0, to: 0}}],
|
|
349
|
-
proxy: {host: "127.0.0.1", port:
|
|
437
|
+
proxy: {host: "127.0.0.1", port: proxyPort},
|
|
350
438
|
...(persistState ? {statePath} : {})
|
|
351
439
|
}
|
|
352
440
|
|
|
@@ -360,7 +448,7 @@ async function createFixture({healthGate = false, healthPath = "/ping", healthTi
|
|
|
360
448
|
}
|
|
361
449
|
|
|
362
450
|
await fs.writeFile(configPath, `${setup}module.exports = ${JSON.stringify(config, null, 2)}\n`)
|
|
363
|
-
return {configPath, gatePath, healthGatePath, lifecyclePath, root, socketPath, startedPath, statePath, stoppedPath}
|
|
451
|
+
return {configPath, gatePath, healthGatePath, lifecyclePath, processPort, proxyPort, root, socketPath, startedPath, statePath, stoppedPath}
|
|
364
452
|
}
|
|
365
453
|
|
|
366
454
|
/**
|
|
@@ -395,6 +483,12 @@ async function waitForFile(filePath) {
|
|
|
395
483
|
const watcher = fs.watch(path.dirname(filePath))
|
|
396
484
|
|
|
397
485
|
try {
|
|
486
|
+
try {
|
|
487
|
+
return await fs.readFile(filePath, "utf8")
|
|
488
|
+
} catch (error) {
|
|
489
|
+
if (!error || typeof error !== "object" || !("code" in error) || error.code !== "ENOENT") throw error
|
|
490
|
+
}
|
|
491
|
+
|
|
398
492
|
for await (const event of watcher) {
|
|
399
493
|
if (event.filename === path.basename(filePath)) return await fs.readFile(filePath, "utf8")
|
|
400
494
|
}
|
|
@@ -420,26 +514,143 @@ function spawnDaemon(fixture, release) {
|
|
|
420
514
|
|
|
421
515
|
/**
|
|
422
516
|
* @param {string[]} args - Daemon arguments.
|
|
517
|
+
* @param {{timeoutMs?: number}} [options] - Process execution options.
|
|
423
518
|
* @returns {Promise<{code: number | null, output: string, stderr: string}>} Completed process result.
|
|
424
519
|
*/
|
|
425
|
-
async function runDaemon(args) {
|
|
426
|
-
return await runRollbridge(["daemon", ...args])
|
|
520
|
+
async function runDaemon(args, options) {
|
|
521
|
+
return await runRollbridge(["daemon", ...args], options)
|
|
427
522
|
}
|
|
428
523
|
|
|
429
524
|
/**
|
|
430
525
|
* @param {string[]} args - Rollbridge command and arguments.
|
|
526
|
+
* @param {{timeoutMs?: number}} [options] - Process execution options.
|
|
431
527
|
* @returns {Promise<{code: number | null, output: string, stderr: string}>} Completed process result.
|
|
432
528
|
*/
|
|
433
|
-
async function runRollbridge(args) {
|
|
529
|
+
async function runRollbridge(args, {timeoutMs} = {}) {
|
|
434
530
|
const child = spawn(process.execPath, [binPath, ...args], {stdio: ["ignore", "pipe", "pipe"]})
|
|
435
531
|
let output = ""
|
|
436
532
|
let stderr = ""
|
|
437
533
|
|
|
438
534
|
child.stdout.setEncoding("utf8").on("data", (chunk) => { output += chunk })
|
|
439
535
|
child.stderr.setEncoding("utf8").on("data", (chunk) => { stderr += chunk })
|
|
440
|
-
|
|
536
|
+
let timer
|
|
537
|
+
|
|
538
|
+
const exited = once(child, "exit")
|
|
539
|
+
const result = timeoutMs === undefined ? await exited : await Promise.race([
|
|
540
|
+
exited,
|
|
541
|
+
new Promise((resolve) => {
|
|
542
|
+
timer = setTimeout(() => {
|
|
543
|
+
child.kill("SIGKILL")
|
|
544
|
+
resolve(["timeout"])
|
|
545
|
+
}, timeoutMs)
|
|
546
|
+
})
|
|
547
|
+
])
|
|
548
|
+
|
|
549
|
+
if (timer) clearTimeout(timer)
|
|
550
|
+
if (result[0] === "timeout") {
|
|
551
|
+
await exited
|
|
552
|
+
throw new Error(`Rollbridge did not terminate within ${timeoutMs}ms`)
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
const [code] = result
|
|
556
|
+
|
|
557
|
+
return {code: typeof code === "number" || code === null ? code : null, output, stderr}
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
/**
|
|
561
|
+
* @param {string} output - JSON-lines daemon output.
|
|
562
|
+
* @returns {StructuredRecord[]} Parsed records.
|
|
563
|
+
*/
|
|
564
|
+
function parseStructuredOutput(output) {
|
|
565
|
+
return output.split("\n").filter(Boolean).map((line) => JSON.parse(line))
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
/**
|
|
569
|
+
* @param {{lifecyclePath: string, processPort: number, proxyPort: number}} fixture - Attempt fixture.
|
|
570
|
+
* @param {StructuredRecord[]} records - Structured daemon records.
|
|
571
|
+
* @returns {Promise<void>} Resolves after all owned resources are verified stopped.
|
|
572
|
+
*/
|
|
573
|
+
async function assertAttemptResourcesStopped(fixture, records) {
|
|
574
|
+
const events = await readLifecycleEvents(fixture.lifecyclePath)
|
|
575
|
+
const started = events.filter((event) => event.event === "started")
|
|
576
|
+
const stoppedPids = new Set(events.filter((event) => event.event === "stopped").map((event) => event.pid))
|
|
577
|
+
const expectedProcessIds = new Set(["database", "scheduler", "web", "worker"])
|
|
578
|
+
const managedStarts = new Set(records.filter((record) => record.message === "process started").map((record) => record.data?.processId))
|
|
579
|
+
const managedExits = new Set(records.filter((record) => record.message === "process exited").map((record) => record.data?.processId))
|
|
580
|
+
|
|
581
|
+
assert.deepEqual(managedStarts, expectedProcessIds)
|
|
582
|
+
assert.deepEqual(managedExits, expectedProcessIds)
|
|
583
|
+
for (const event of started) {
|
|
584
|
+
assert.equal(stoppedPids.has(event.pid), true, `${event.processId} must receive graceful shutdown`)
|
|
585
|
+
assert.equal(isProcessAlive(Number(event.pid)), false, `${event.processId} pid ${event.pid} must be gone before daemon exit`)
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
await assertPortAvailable(fixture.processPort)
|
|
589
|
+
await assertPortAvailable(fixture.proxyPort)
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
/**
|
|
593
|
+
* @param {string} lifecyclePath - Fixture lifecycle log.
|
|
594
|
+
* @returns {Promise<Record<string, import("../src/json.js").JsonValue>[]>} Parsed lifecycle events.
|
|
595
|
+
*/
|
|
596
|
+
async function readLifecycleEvents(lifecyclePath) {
|
|
597
|
+
return (await fs.readFile(lifecyclePath, "utf8")).trim().split("\n").filter(Boolean).map((line) => JSON.parse(line))
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
/**
|
|
601
|
+
* Best-effort cleanup for a deliberately failing lingering-process regression.
|
|
602
|
+
* @param {string} lifecyclePath - Fixture lifecycle log.
|
|
603
|
+
* @returns {Promise<void>} Resolves after known fixture process groups are stopped.
|
|
604
|
+
*/
|
|
605
|
+
async function killAttemptProcesses(lifecyclePath) {
|
|
606
|
+
try {
|
|
607
|
+
const events = await readLifecycleEvents(lifecyclePath)
|
|
608
|
+
|
|
609
|
+
for (const event of events) {
|
|
610
|
+
const pid = Number(event.pid)
|
|
611
|
+
|
|
612
|
+
if (event.event === "started" && isProcessAlive(pid)) process.kill(-pid, "SIGKILL")
|
|
613
|
+
}
|
|
614
|
+
} catch {
|
|
615
|
+
// The attempt can fail before creating the lifecycle file.
|
|
616
|
+
}
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
/**
|
|
620
|
+
* @param {number} port - Local port expected to be free.
|
|
621
|
+
* @returns {Promise<void>} Resolves after binding and releasing the port.
|
|
622
|
+
*/
|
|
623
|
+
async function assertPortAvailable(port) {
|
|
624
|
+
const server = net.createServer()
|
|
625
|
+
|
|
626
|
+
await new Promise((resolve, reject) => {
|
|
627
|
+
server.once("error", reject)
|
|
628
|
+
server.listen(port, "127.0.0.1", () => resolve(undefined))
|
|
629
|
+
})
|
|
630
|
+
await new Promise((resolve) => server.close(() => resolve(undefined)))
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
/**
|
|
634
|
+
* Reserves distinct ephemeral ports until all have been observed, then releases them.
|
|
635
|
+
* @param {number} count - Number of ports.
|
|
636
|
+
* @returns {Promise<number[]>} Available port numbers.
|
|
637
|
+
*/
|
|
638
|
+
async function availablePorts(count) {
|
|
639
|
+
const servers = Array.from({length: count}, () => net.createServer())
|
|
640
|
+
|
|
641
|
+
await Promise.all(servers.map((server) => new Promise((resolve, reject) => {
|
|
642
|
+
server.once("error", reject)
|
|
643
|
+
server.listen(0, "127.0.0.1", () => resolve(undefined))
|
|
644
|
+
})))
|
|
645
|
+
const ports = servers.map((server) => {
|
|
646
|
+
const address = server.address()
|
|
647
|
+
|
|
648
|
+
if (!address || typeof address === "string") throw new Error("Expected a TCP server address")
|
|
649
|
+
return address.port
|
|
650
|
+
})
|
|
441
651
|
|
|
442
|
-
|
|
652
|
+
await Promise.all(servers.map((server) => new Promise((resolve) => server.close(() => resolve(undefined)))))
|
|
653
|
+
return ports
|
|
443
654
|
}
|
|
444
655
|
|
|
445
656
|
/**
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
import fs from "node:fs"
|
|
4
|
+
|
|
5
|
+
const lifecyclePath = process.env.ROLLBRIDGE_TEST_LIFECYCLE_PATH
|
|
6
|
+
|
|
7
|
+
if (!lifecyclePath) throw new Error("ROLLBRIDGE_TEST_LIFECYCLE_PATH is required")
|
|
8
|
+
|
|
9
|
+
/** @param {string} event - Lifecycle event. */
|
|
10
|
+
const record = (event) => {
|
|
11
|
+
fs.appendFileSync(lifecyclePath, `${JSON.stringify({event, pid: process.pid, processId: process.env.ROLLBRIDGE_PROCESS_ID})}\n`)
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
record("started")
|
|
15
|
+
process.on("SIGTERM", () => {
|
|
16
|
+
record("stopped")
|
|
17
|
+
process.exit(0)
|
|
18
|
+
})
|
|
19
|
+
setInterval(() => {}, 1000)
|
|
@@ -401,18 +401,18 @@ test("sends the configured stopSignal as the graceful stop signal", async () =>
|
|
|
401
401
|
|
|
402
402
|
/** @type {string[]} */
|
|
403
403
|
const signals = []
|
|
404
|
-
const
|
|
404
|
+
const killProcess = managed.killProcess.bind(managed)
|
|
405
405
|
|
|
406
|
-
managed.
|
|
406
|
+
managed.killProcess = (pid, signal) => {
|
|
407
407
|
signals.push(signal)
|
|
408
|
-
|
|
408
|
+
killProcess(pid, signal)
|
|
409
409
|
}
|
|
410
410
|
|
|
411
411
|
await managed.start()
|
|
412
412
|
await managed.stop()
|
|
413
413
|
|
|
414
414
|
// The graceful stop uses the configured signal (a SIGKILL fallback, if any, comes after).
|
|
415
|
-
assert.
|
|
415
|
+
assert.deepEqual(signals, ["SIGINT", "SIGINT"])
|
|
416
416
|
assert.equal(managed.status().state, "stopped")
|
|
417
417
|
})
|
|
418
418
|
|
|
@@ -431,11 +431,11 @@ test("indefinite stop waits for the process to exit without SIGKILL", async () =
|
|
|
431
431
|
})
|
|
432
432
|
/** @type {string[]} */
|
|
433
433
|
const signals = []
|
|
434
|
-
const
|
|
434
|
+
const killProcess = managed.killProcess.bind(managed)
|
|
435
435
|
|
|
436
|
-
managed.
|
|
436
|
+
managed.killProcess = (pid, signal) => {
|
|
437
437
|
signals.push(signal)
|
|
438
|
-
|
|
438
|
+
killProcess(pid, signal)
|
|
439
439
|
}
|
|
440
440
|
|
|
441
441
|
try {
|
|
@@ -443,7 +443,7 @@ test("indefinite stop waits for the process to exit without SIGKILL", async () =
|
|
|
443
443
|
await managed.stop()
|
|
444
444
|
|
|
445
445
|
assert.equal(managed.status().state, "stopped")
|
|
446
|
-
assert.deepEqual(signals, ["SIGTERM"])
|
|
446
|
+
assert.deepEqual(signals, ["SIGTERM", "SIGTERM"])
|
|
447
447
|
} finally {
|
|
448
448
|
await managed.stop()
|
|
449
449
|
}
|
|
@@ -491,6 +491,42 @@ test("stop waits for process group descendants after the detached shell exits",
|
|
|
491
491
|
}
|
|
492
492
|
})
|
|
493
493
|
|
|
494
|
+
test("stop does not return while a gracefully stopped descendant remains unreaped", async () => {
|
|
495
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-process-group-"))
|
|
496
|
+
const pidPath = path.join(dir, "child.pid")
|
|
497
|
+
const child = [
|
|
498
|
+
"const fs = require('node:fs')",
|
|
499
|
+
`fs.writeFileSync(${JSON.stringify(pidPath)}, String(process.pid))`,
|
|
500
|
+
"process.on('SIGTERM', () => setTimeout(() => process.exit(0), 50))",
|
|
501
|
+
"setInterval(() => {}, 1000)"
|
|
502
|
+
].join("; ")
|
|
503
|
+
const managed = new ManagedProcess({
|
|
504
|
+
command: `trap 'exit 0' TERM; ${JSON.stringify(process.execPath)} -e ${JSON.stringify(child)} & wait`,
|
|
505
|
+
cwd: undefined,
|
|
506
|
+
env: {},
|
|
507
|
+
id: "worker",
|
|
508
|
+
logger: () => {},
|
|
509
|
+
outputLines: 50,
|
|
510
|
+
restartDelayMs: 10,
|
|
511
|
+
shouldRestart: () => false,
|
|
512
|
+
stopSignal: "SIGTERM",
|
|
513
|
+
stopTimeoutMs: 2000
|
|
514
|
+
})
|
|
515
|
+
|
|
516
|
+
try {
|
|
517
|
+
await managed.start()
|
|
518
|
+
await waitFor(() => fs.existsSync(pidPath))
|
|
519
|
+
const childPid = Number(fs.readFileSync(pidPath, "utf8"))
|
|
520
|
+
|
|
521
|
+
await managed.stop()
|
|
522
|
+
|
|
523
|
+
assert.throws(() => process.kill(childPid, 0), {code: "ESRCH"})
|
|
524
|
+
} finally {
|
|
525
|
+
await managed.stop()
|
|
526
|
+
fs.rmSync(dir, {force: true, recursive: true})
|
|
527
|
+
}
|
|
528
|
+
})
|
|
529
|
+
|
|
494
530
|
test("a memory restart respawns and is counted when the supervisor still wants the process", async () => {
|
|
495
531
|
const managed = buildLongLived(() => true)
|
|
496
532
|
|
package/test/rollbridge.test.js
CHANGED
|
@@ -855,6 +855,14 @@ test("a process over its memory limit is restarted with reason memory", {skip: p
|
|
|
855
855
|
assert.equal(hog.lastStartReason, "memory")
|
|
856
856
|
assert.equal(typeof hog.lastMemoryRestartAt, "string")
|
|
857
857
|
|
|
858
|
+
// Keep the replacement alive long enough to observe its next monitor sample. The fixture
|
|
859
|
+
// remains over the configured limit after every launch, otherwise it can restart again and
|
|
860
|
+
// clear rssBytes/children before this polling loop observes them on slower CI runners.
|
|
861
|
+
const hogProcess = daemon.activeRelease?.processes.get("hog")
|
|
862
|
+
|
|
863
|
+
assert.ok(hogProcess?.memory)
|
|
864
|
+
hogProcess.memory.limitBytes = Number.MAX_SAFE_INTEGER
|
|
865
|
+
|
|
858
866
|
// rssBytes is sampled on the monitor's interval; wait for a measurement of the running process.
|
|
859
867
|
await waitFor(() => {
|
|
860
868
|
const rssBytes = activeProcessStatus(daemon, "hog")?.rssBytes
|
|
@@ -115,6 +115,43 @@ test("shutdown response waits for endpoint and owned-process cleanup before imme
|
|
|
115
115
|
}
|
|
116
116
|
})
|
|
117
117
|
|
|
118
|
+
test("external-owner retirement releases listeners before a long-draining companion exits", async () => {
|
|
119
|
+
const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-owner-retirement-"))
|
|
120
|
+
const socketPath = path.join(root, "rollbridge.sock")
|
|
121
|
+
const gatePath = path.join(root, "retire.fifo")
|
|
122
|
+
const stoppingPath = path.join(root, "stopping")
|
|
123
|
+
const gate = spawn("mkfifo", [gatePath])
|
|
124
|
+
assert.equal((await once(gate, "exit"))[0], 0)
|
|
125
|
+
const config = buildConfig(socketPath, {companion: {
|
|
126
|
+
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify("setInterval(() => {}, 1000)")}`,
|
|
127
|
+
id: "worker",
|
|
128
|
+
lifecycle: {drainCommand: `read released < ${JSON.stringify(gatePath)}`, drainTimeoutMs: 60000, quietCommand: `printf stopping > ${JSON.stringify(stoppingPath)}`},
|
|
129
|
+
policy: "companion"
|
|
130
|
+
}})
|
|
131
|
+
const daemon = new RollbridgeDaemon({config, logger: () => {}})
|
|
132
|
+
let replacement
|
|
133
|
+
|
|
134
|
+
try {
|
|
135
|
+
await daemon.start()
|
|
136
|
+
await daemon.deploy({releaseId: "old", releasePath: root, revision: "old"})
|
|
137
|
+
const retirement = sendControlCommand({command: {attestation: `sha256:${"a".repeat(64)}`, command: "retire-owner"}, path: socketPath})
|
|
138
|
+
const response = await retirement
|
|
139
|
+
assert.deepEqual(response, {message: "owner retired", status: "success"})
|
|
140
|
+
assert.equal(await fs.readFile(stoppingPath, "utf8"), "stopping", "old worker must stop accepting work before listener takeover")
|
|
141
|
+
|
|
142
|
+
replacement = new RollbridgeDaemon({config, logger: () => {}})
|
|
143
|
+
await replacement.start({reportOrphans: false})
|
|
144
|
+
assert.equal((await sendControlCommand({command: {command: "status"}, path: socketPath})).application, "shutdown-target")
|
|
145
|
+
assert.deepEqual(replacement.status().orphans, [], "intentional retired companions are not replacement orphans")
|
|
146
|
+
assert.equal(daemon.status().releases[0].processes[0].state, "stopping")
|
|
147
|
+
} finally {
|
|
148
|
+
await fs.writeFile(gatePath, "done\n").catch(() => {})
|
|
149
|
+
if (replacement) await replacement.shutdown()
|
|
150
|
+
await daemon.shutdown()
|
|
151
|
+
await fs.rm(root, {force: true, recursive: true})
|
|
152
|
+
}
|
|
153
|
+
})
|
|
154
|
+
|
|
118
155
|
test("control socket unlink failure is reported only after owned cleanup completes", async () => {
|
|
119
156
|
const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-shutdown-unlink-failure-"))
|
|
120
157
|
const socketPath = path.join(root, "control.sock")
|