rollbridge 0.1.28 → 0.1.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +14 -0
- package/README.md +69 -14
- package/TODO.md +5 -2
- package/changelog.d/20260828-atomic-owner-replacement.md +22 -0
- package/changelog.d/20260828-durable-owner-recovery.md +7 -0
- package/changelog.d/20260828-same-owner-jobs-generations.md +6 -0
- package/docs/cli.md +43 -21
- package/docs/config.md +71 -18
- package/docs/logging.md +8 -3
- package/docs/tensorbuzz-runbook.md +7 -6
- package/docs/troubleshooting.md +28 -12
- package/docs/velocious.md +11 -4
- package/docs/workers.md +8 -2
- package/examples/tensorbuzz.com.js +12 -4
- package/package.json +1 -1
- package/src/cli.js +209 -36
- package/src/config.js +8 -2
- package/src/control-client.js +118 -1
- package/src/daemon.js +939 -53
- package/src/guardian-client.js +434 -0
- package/src/managed-process.js +45 -15
- package/src/process-guardian.js +601 -0
- package/src/release-group.js +190 -15
- package/src/state-store.js +1 -1
- package/test/config-validation.test.js +22 -0
- package/test/fixtures/pre-split3-daemon-runner.js +30 -0
- package/test/fixtures/pre-split3-daemon.js +1336 -0
- package/test/fixtures/pre-split3-guardian-client.js +293 -0
- package/test/fixtures/pre-split3-process-guardian.js +292 -0
- package/test/fixtures/service-app.js +32 -2
- package/test/guardian-client.test.js +304 -0
- package/test/owner-recovery.test.js +950 -0
- package/test/owner-replacement.test.js +772 -0
- package/test/release-runtime-retention.test.js +1 -1
- package/test/rollbridge.test.js +178 -5
- package/test/shutdown-completion.test.js +1 -1
- package/test/state-store.test.js +12 -0
package/docs/velocious.md
CHANGED
|
@@ -26,6 +26,7 @@ export default {
|
|
|
26
26
|
application: "tensorbuzz",
|
|
27
27
|
control: {path: "/tmp/rollbridge-tensorbuzz.sock"},
|
|
28
28
|
statePath: "/var/lib/rollbridge/tensorbuzz.json",
|
|
29
|
+
ownerRecovery: {reconnectGraceMs: 30000},
|
|
29
30
|
|
|
30
31
|
proxy: {
|
|
31
32
|
host: "127.0.0.1",
|
|
@@ -56,6 +57,7 @@ export default {
|
|
|
56
57
|
VELOCIOUS_BACKGROUND_JOBS_PORT: "{{port}}"
|
|
57
58
|
},
|
|
58
59
|
command: "wait-for-it 127.0.0.1:{{ports.beacon}} --strict -- npx velocious background-jobs-main",
|
|
60
|
+
lifecycle: {quietCommand: "appctl jobs-main-retire --pid $ROLLBRIDGE_PID"},
|
|
59
61
|
port: {from: 7331, to: 7399}
|
|
60
62
|
},
|
|
61
63
|
{
|
|
@@ -89,6 +91,9 @@ export default {
|
|
|
89
91
|
}
|
|
90
92
|
```
|
|
91
93
|
|
|
94
|
+
`appctl` is a placeholder for a reviewed Velocious/application integration that
|
|
95
|
+
quiesces jobs-main admission without terminating its worker/report endpoint.
|
|
96
|
+
|
|
92
97
|
Beacon keeps its fixed port because it is intentionally shared. Jobs-main uses a
|
|
93
98
|
range because every release gets its own coordinator. Same-release
|
|
94
99
|
`{{ports.background-jobs-main}}` expansion ensures that old workers retain the
|
|
@@ -138,10 +143,12 @@ report every referenced release directory so Rampway can pin it against cleanup.
|
|
|
138
143
|
A runtime owner/version handoff preserves or transfers that supervision and
|
|
139
144
|
returns after the replacement is healthy; it is not a full synchronous shutdown.
|
|
140
145
|
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
146
|
+
Rollbridge implements post-activation quiescence, concurrent endpoints,
|
|
147
|
+
asynchronous generation drain, and `status.releaseReferences`. With
|
|
148
|
+
`ownerRecovery`, a same-authority replacement reconnects to the guardian rather
|
|
149
|
+
than adopting arbitrary PIDs, and `ensure-daemon` can atomically transfer
|
|
150
|
+
incompatible owner/config/control-socket/package/runtime authority while the
|
|
151
|
+
guardian retains every generation. `--takeover-owner` is not this handoff.
|
|
145
152
|
|
|
146
153
|
## Timeouts
|
|
147
154
|
|
package/docs/workers.md
CHANGED
|
@@ -17,6 +17,7 @@ fixed port such as `7330`.
|
|
|
17
17
|
policy: "service",
|
|
18
18
|
deployStrategy: "handoff",
|
|
19
19
|
command: "npx velocious background-jobs-main",
|
|
20
|
+
lifecycle: {quietCommand: "appctl jobs-main-retire --pid $ROLLBRIDGE_PID"},
|
|
20
21
|
port: {from: 7331, to: 7399}
|
|
21
22
|
},
|
|
22
23
|
{
|
|
@@ -30,6 +31,9 @@ fixed port such as `7330`.
|
|
|
30
31
|
}
|
|
31
32
|
```
|
|
32
33
|
|
|
34
|
+
The illustrative `appctl` command must be replaced by the application's real,
|
|
35
|
+
reviewed jobs-main quiescence control.
|
|
36
|
+
|
|
33
37
|
Each worker receives its generation's jobs-main port. Old workers keep that port
|
|
34
38
|
for their entire lifetime; normal deploy draining never hands them to, or lets
|
|
35
39
|
them reconnect to, the new jobs-main. `replicas` scales the pool as
|
|
@@ -67,8 +71,10 @@ HTTP/WebSocket connections, or other retained services to finish. The required
|
|
|
67
71
|
supervisor contract durably retains generations after the command returns and
|
|
68
72
|
across later deploys and supervisor/host recovery. Every referenced release
|
|
69
73
|
directory must be reported to Rampway and stays pinned against cleanup until the
|
|
70
|
-
last retained process exits.
|
|
71
|
-
|
|
74
|
+
last retained process exits. Rollbridge exposes references as
|
|
75
|
+
`status.releaseReferences`, supports concurrent retired generations, and with
|
|
76
|
+
`ownerRecovery` preserves them across same-authority daemon recovery and atomic
|
|
77
|
+
incompatible `ensure-daemon` owner replacement; see
|
|
72
78
|
[`docs/config.md`](config.md#processesdeploystrategy) and
|
|
73
79
|
[`docs/cli.md`](cli.md#deploy).
|
|
74
80
|
|
|
@@ -2,8 +2,9 @@
|
|
|
2
2
|
//
|
|
3
3
|
// Nginx should keep proxying the backend host to 127.0.0.1:4500. Rollbridge
|
|
4
4
|
// binds that stable HTTP port, forwards to the active release's internal web
|
|
5
|
-
// port
|
|
6
|
-
//
|
|
5
|
+
// port and keeps Beacon daemon-wide. The jobs-main and worker controls below
|
|
6
|
+
// are illustrative: replace appctl with application commands that quiesce new
|
|
7
|
+
// admission without terminating the generation.
|
|
7
8
|
|
|
8
9
|
export default {
|
|
9
10
|
application: "tensorbuzz",
|
|
@@ -12,6 +13,9 @@ export default {
|
|
|
12
13
|
path: "/tmp/rollbridge-tensorbuzz.sock"
|
|
13
14
|
},
|
|
14
15
|
|
|
16
|
+
statePath: "/var/lib/rollbridge/tensorbuzz.json",
|
|
17
|
+
ownerRecovery: {reconnectGraceMs: 30000},
|
|
18
|
+
|
|
15
19
|
proxy: {
|
|
16
20
|
host: "127.0.0.1",
|
|
17
21
|
port: 4500,
|
|
@@ -37,6 +41,7 @@ export default {
|
|
|
37
41
|
{
|
|
38
42
|
id: "background-jobs-main",
|
|
39
43
|
policy: "service",
|
|
44
|
+
deployStrategy: "handoff",
|
|
40
45
|
cwd: "{{releasePath}}/backend",
|
|
41
46
|
env: {
|
|
42
47
|
NODE_ENV: "production",
|
|
@@ -45,7 +50,8 @@ export default {
|
|
|
45
50
|
VELOCIOUS_BACKGROUND_JOBS_PORT: "{{port}}"
|
|
46
51
|
},
|
|
47
52
|
command: "wait-for-it 127.0.0.1:{{ports.beacon}} --strict -- npx velocious background-jobs-main",
|
|
48
|
-
|
|
53
|
+
lifecycle: {quietCommand: "appctl jobs-main-retire --pid $ROLLBRIDGE_PID"},
|
|
54
|
+
port: {from: 7331, to: 7399}
|
|
49
55
|
},
|
|
50
56
|
{
|
|
51
57
|
id: "background-jobs-worker",
|
|
@@ -58,7 +64,9 @@ export default {
|
|
|
58
64
|
VELOCIOUS_BACKGROUND_JOBS_PORT: "{{ports.background-jobs-main}}"
|
|
59
65
|
},
|
|
60
66
|
command: "wait-for-it 127.0.0.1:{{ports.beacon}} --strict -- wait-for-it 127.0.0.1:{{ports.background-jobs-main}} --strict -- npx velocious background-jobs-worker",
|
|
61
|
-
|
|
67
|
+
lifecycle: {quietCommand: "appctl jobs-worker-retire --pid $ROLLBRIDGE_PID"},
|
|
68
|
+
nonBlockingDrain: true,
|
|
69
|
+
gracefulStopMs: "indefinite"
|
|
62
70
|
},
|
|
63
71
|
{
|
|
64
72
|
id: "web",
|
package/package.json
CHANGED
package/src/cli.js
CHANGED
|
@@ -5,17 +5,20 @@ import fsPromises from "node:fs/promises"
|
|
|
5
5
|
import {createHash} from "node:crypto"
|
|
6
6
|
import path from "node:path"
|
|
7
7
|
import {spawn} from "node:child_process"
|
|
8
|
-
import {Command} from "commander"
|
|
9
|
-
import RollbridgeDaemon from "./daemon.js"
|
|
8
|
+
import {Command, Option} from "commander"
|
|
9
|
+
import RollbridgeDaemon, {ownerConfigDigest} from "./daemon.js"
|
|
10
10
|
import {loadDaemonRuntimeIdentity, prepareDaemonRuntime} from "./daemon-runtime.js"
|
|
11
11
|
import {loadConfig, parseConfigFile, resolveConfigPath, validateConfig} from "./config.js"
|
|
12
12
|
import {runEnvironmentChecks, runReleaseChecks} from "./doctor.js"
|
|
13
13
|
import {predeployCleanup} from "./predeploy-cleanup.js"
|
|
14
14
|
import {recoverOrphans} from "./recover.js"
|
|
15
15
|
import {sendControlCommand} from "./control-client.js"
|
|
16
|
+
import {readState} from "./state-store.js"
|
|
16
17
|
|
|
17
18
|
const DEFAULT_DAEMON_START_TIMEOUT_MS = 10000
|
|
18
19
|
|
|
20
|
+
/** @typedef {import("node:child_process").ChildProcess["signalCode"]} DaemonExitSignal */
|
|
21
|
+
|
|
19
22
|
/**
|
|
20
23
|
* Runs the CLI.
|
|
21
24
|
* @param {string[]} argv - Process argv.
|
|
@@ -37,16 +40,45 @@ export async function runCli(argv) {
|
|
|
37
40
|
.option("--revision <sha>", "Bootstrap revision (requires --config, --release-path, and --release-id)")
|
|
38
41
|
.option("--boot-attestation <digest>", "Opaque bootstrap ownership attestation (requires the complete bootstrap release tuple)")
|
|
39
42
|
.option("--takeover-owner", "Boot and health-check before retiring the current external owner")
|
|
43
|
+
.option("--replace-owner", "Resume a prepared durable owner replacement")
|
|
44
|
+
.addOption(new Option("--legacy-incumbent-pid <pid>").hideHelp())
|
|
40
45
|
.action(async (options) => {
|
|
41
46
|
const bootstrap = await validateDaemonBootstrapOptions(options)
|
|
42
47
|
const configPath = await resolveConfigPath(options.config)
|
|
43
48
|
const config = await loadConfig(configPath)
|
|
44
49
|
const runtime = await loadDaemonRuntimeIdentity(process.env.ROLLBRIDGE_DAEMON_RUNTIME_MANIFEST)
|
|
45
|
-
const daemon = new RollbridgeDaemon({
|
|
50
|
+
const daemon = new RollbridgeDaemon({
|
|
51
|
+
bootstrap,
|
|
52
|
+
config,
|
|
53
|
+
configPath,
|
|
54
|
+
legacyIncumbentPid: positiveIntegerOrUndefined(options.legacyIncumbentPid, "legacy incumbent pid"),
|
|
55
|
+
runtime
|
|
56
|
+
})
|
|
46
57
|
|
|
47
58
|
if (options.takeoverOwner && (!bootstrap || !bootstrap.attestation)) throw new Error("Daemon --takeover-owner requires the complete bootstrap release tuple and --boot-attestation.")
|
|
48
59
|
|
|
49
|
-
if (
|
|
60
|
+
if (options.replaceOwner) {
|
|
61
|
+
if (bootstrap || options.takeoverOwner) throw new Error("Daemon --replace-owner cannot be combined with bootstrap takeover options.")
|
|
62
|
+
await daemon.replaceIncompatibleOwner()
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
if (!options.takeoverOwner && !options.replaceOwner) {
|
|
66
|
+
try {
|
|
67
|
+
await daemon.start({exposeControl: !bootstrap})
|
|
68
|
+
} catch (error) {
|
|
69
|
+
if (!config.ownerRecovery) throw error
|
|
70
|
+
|
|
71
|
+
const winner = await sendControlCommand({command: {command: "status"}, path: config.control.path}).catch(() => undefined)
|
|
72
|
+
const matchingWinner = winner?.application === config.application &&
|
|
73
|
+
winner.ownerRecovery && typeof winner.ownerRecovery === "object" && !Array.isArray(winner.ownerRecovery) &&
|
|
74
|
+
winner.ownerRecovery.configDigest === daemon.ownerRecoveryConfigDigest() &&
|
|
75
|
+
((!runtime && !winner.daemonRuntime) || (runtime && winner.daemonRuntime && typeof winner.daemonRuntime === "object" && !Array.isArray(winner.daemonRuntime) && winner.daemonRuntime.digest === runtime.digest))
|
|
76
|
+
|
|
77
|
+
if (!matchingWinner) throw error
|
|
78
|
+
await daemon.abandonOwnerRecoveryAttempt()
|
|
79
|
+
return
|
|
80
|
+
}
|
|
81
|
+
}
|
|
50
82
|
|
|
51
83
|
const shutdown = async () => {
|
|
52
84
|
await daemon.shutdown()
|
|
@@ -70,6 +102,11 @@ export async function runCli(argv) {
|
|
|
70
102
|
|
|
71
103
|
daemon.logger("bootstrap activation failed", {releaseId: bootstrap.releaseId, status: "error", ...errorLogData(failure)})
|
|
72
104
|
|
|
105
|
+
if (config.ownerRecovery && daemon.activeRelease) {
|
|
106
|
+
await daemon.exposeControl()
|
|
107
|
+
return
|
|
108
|
+
}
|
|
109
|
+
|
|
73
110
|
try {
|
|
74
111
|
await daemon.shutdown()
|
|
75
112
|
} catch (shutdownError) {
|
|
@@ -798,23 +835,56 @@ async function validateDaemonBootstrapOptions(options) {
|
|
|
798
835
|
async function ensureDaemonRunning({config, configPath, logPath, pidPath, runtimePath, timeoutMs}) {
|
|
799
836
|
const runtime = await prepareDaemonRuntime(runtimePath || defaultDaemonRuntimePath(config))
|
|
800
837
|
const existingStatus = await daemonStatus(config)
|
|
838
|
+
const expectedConfigDigest = ownerConfigDigest(config)
|
|
801
839
|
|
|
802
840
|
if (existingStatus) {
|
|
803
|
-
|
|
804
|
-
|
|
841
|
+
const matchingRecoveryAuthority = !config.ownerRecovery || (
|
|
842
|
+
existingStatus.ownerRecovery && typeof existingStatus.ownerRecovery === "object" && !Array.isArray(existingStatus.ownerRecovery) &&
|
|
843
|
+
existingStatus.ownerRecovery.configDigest === expectedConfigDigest
|
|
844
|
+
)
|
|
845
|
+
|
|
846
|
+
if (compatibleDaemonRuntime(existingStatus, runtime) && matchingRecoveryAuthority) return existingStatus
|
|
847
|
+
if (!config.ownerRecovery) assertCompatibleDaemonRuntime(existingStatus, runtime)
|
|
805
848
|
}
|
|
806
849
|
|
|
807
|
-
await
|
|
850
|
+
const persistedState = config.ownerRecovery && config.statePath ? await readState(config.statePath) : undefined
|
|
851
|
+
const persistedOwner = persistedState && typeof persistedState === "object" && !Array.isArray(persistedState) ? persistedState : undefined
|
|
852
|
+
const persistedRecovery = persistedOwner?.recovery
|
|
853
|
+
const replacement = Boolean(config.ownerRecovery && (existingStatus || (persistedRecovery && typeof persistedRecovery === "object" && !Array.isArray(persistedRecovery) && (
|
|
854
|
+
persistedRecovery.configDigest !== expectedConfigDigest || !compatibleDaemonRuntime(/** @type {Record<string, import("./json.js").JsonValue>} */ (persistedOwner), runtime)
|
|
855
|
+
))))
|
|
856
|
+
const resolvedPidPath = pidPath || defaultDaemonPidPath(config)
|
|
857
|
+
const legacyIncumbentPid = replacement ? await readDaemonPid(resolvedPidPath) : undefined
|
|
858
|
+
|
|
859
|
+
await fsPromises.mkdir(path.dirname(resolvedPidPath), {recursive: true})
|
|
860
|
+
const candidate = await startDaemonProcess({
|
|
808
861
|
configPath,
|
|
809
862
|
logPath: logPath || defaultDaemonLogPath(config),
|
|
810
|
-
|
|
863
|
+
replacement,
|
|
864
|
+
legacyIncumbentPid,
|
|
811
865
|
runtime
|
|
812
866
|
})
|
|
867
|
+
let startedStatus
|
|
813
868
|
|
|
814
|
-
|
|
869
|
+
try {
|
|
870
|
+
startedStatus = await waitForDaemonStatus(config, timeoutMs, {
|
|
871
|
+
candidate,
|
|
872
|
+
configDigest: config.ownerRecovery ? expectedConfigDigest : undefined,
|
|
873
|
+
runtime
|
|
874
|
+
})
|
|
815
875
|
|
|
816
|
-
|
|
817
|
-
|
|
876
|
+
assertCompatibleDaemonRuntime(startedStatus, runtime)
|
|
877
|
+
const startedPid = positiveIntegerOrUndefined(
|
|
878
|
+
typeof startedStatus.daemonPid === "number" ? String(startedStatus.daemonPid) : undefined,
|
|
879
|
+
"started daemon status PID"
|
|
880
|
+
)
|
|
881
|
+
|
|
882
|
+
if (!startedPid) throw new Error("Started Rollbridge daemon did not report its exact PID")
|
|
883
|
+
await fsPromises.writeFile(resolvedPidPath, `${startedPid}\n`)
|
|
884
|
+
return startedStatus
|
|
885
|
+
} finally {
|
|
886
|
+
candidate.unref()
|
|
887
|
+
}
|
|
818
888
|
}
|
|
819
889
|
|
|
820
890
|
/**
|
|
@@ -843,60 +913,156 @@ async function daemonStatus(config) {
|
|
|
843
913
|
* @param {object} args - Options.
|
|
844
914
|
* @param {string} args.configPath - Config path.
|
|
845
915
|
* @param {string} args.logPath - Log file path.
|
|
846
|
-
* @param {
|
|
916
|
+
* @param {boolean} args.replacement - Whether to run the incompatible replacement transaction.
|
|
917
|
+
* @param {number | undefined} args.legacyIncumbentPid - Exact incumbent recorded before candidate spawn.
|
|
847
918
|
* @param {import("./daemon-runtime.js").DaemonRuntimeIdentity} args.runtime - Prepared runtime.
|
|
848
|
-
* @returns {Promise<
|
|
919
|
+
* @returns {Promise<import("node:child_process").ChildProcess>} Referenced child after exact spawn completion.
|
|
849
920
|
*/
|
|
850
|
-
async function startDaemonProcess({configPath, logPath,
|
|
921
|
+
async function startDaemonProcess({configPath, legacyIncumbentPid, logPath, replacement = false, runtime}) {
|
|
851
922
|
await fsPromises.mkdir(path.dirname(logPath), {recursive: true})
|
|
852
|
-
await fsPromises.mkdir(path.dirname(pidPath), {recursive: true})
|
|
853
923
|
|
|
854
924
|
const stdoutFd = fs.openSync(logPath, "a")
|
|
855
925
|
const stderrFd = fs.openSync(logPath, "a")
|
|
856
926
|
|
|
857
927
|
try {
|
|
858
|
-
const child = spawn(process.execPath, [
|
|
928
|
+
const child = spawn(process.execPath, [
|
|
929
|
+
path.join(runtime.path, "bin", "rollbridge"), "daemon", "--config", configPath,
|
|
930
|
+
...(replacement ? ["--replace-owner"] : []),
|
|
931
|
+
...(legacyIncumbentPid ? ["--legacy-incumbent-pid", String(legacyIncumbentPid)] : [])
|
|
932
|
+
], {
|
|
859
933
|
detached: true,
|
|
860
934
|
env: {...process.env, ROLLBRIDGE_DAEMON_RUNTIME_MANIFEST: path.join(runtime.path, "runtime.json")},
|
|
861
935
|
stdio: ["ignore", stdoutFd, stderrFd]
|
|
862
936
|
})
|
|
863
937
|
|
|
864
|
-
|
|
938
|
+
await new Promise((resolve, reject) => {
|
|
939
|
+
const onError = (/** @type {Error} */ error) => finish(error)
|
|
940
|
+
const onSpawn = () => finish(undefined)
|
|
941
|
+
const finish = (/** @type {Error | undefined} */ error) => {
|
|
942
|
+
child.off("error", onError)
|
|
943
|
+
child.off("spawn", onSpawn)
|
|
944
|
+
if (error) reject(error)
|
|
945
|
+
else resolve(undefined)
|
|
946
|
+
}
|
|
865
947
|
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
}
|
|
948
|
+
child.once("error", onError)
|
|
949
|
+
child.once("spawn", onSpawn)
|
|
950
|
+
})
|
|
951
|
+
return child
|
|
869
952
|
} finally {
|
|
870
953
|
fs.closeSync(stdoutFd)
|
|
871
954
|
fs.closeSync(stderrFd)
|
|
872
955
|
}
|
|
873
956
|
}
|
|
874
957
|
|
|
958
|
+
/**
|
|
959
|
+
* Reads an exact daemon PID before the candidate overwrites the PID file.
|
|
960
|
+
* @param {string} pidPath - Daemon PID file.
|
|
961
|
+
* @returns {Promise<number | undefined>} Positive PID when the file exists and is valid.
|
|
962
|
+
*/
|
|
963
|
+
async function readDaemonPid(pidPath) {
|
|
964
|
+
let value
|
|
965
|
+
|
|
966
|
+
try {
|
|
967
|
+
value = await fsPromises.readFile(pidPath, "utf8")
|
|
968
|
+
} catch (error) {
|
|
969
|
+
if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") return undefined
|
|
970
|
+
throw error
|
|
971
|
+
}
|
|
972
|
+
const trimmed = value.trim()
|
|
973
|
+
|
|
974
|
+
if (!/^\d+$/.test(trimmed)) throw new Error(`Daemon PID file ${pidPath} does not contain one positive integer`)
|
|
975
|
+
return positiveIntegerOrUndefined(trimmed, `daemon PID file ${pidPath}`)
|
|
976
|
+
}
|
|
977
|
+
|
|
978
|
+
/**
|
|
979
|
+
* @param {string | undefined} value - Integer text.
|
|
980
|
+
* @param {string} label - Diagnostic label.
|
|
981
|
+
* @returns {number | undefined} Positive integer or undefined.
|
|
982
|
+
*/
|
|
983
|
+
function positiveIntegerOrUndefined(value, label) {
|
|
984
|
+
if (value === undefined) return undefined
|
|
985
|
+
const parsed = Number(value)
|
|
986
|
+
|
|
987
|
+
if (!Number.isSafeInteger(parsed) || parsed < 1) throw new Error(`${label} must be a positive integer`)
|
|
988
|
+
return parsed
|
|
989
|
+
}
|
|
990
|
+
|
|
875
991
|
/**
|
|
876
992
|
* Waits until a daemon answers status commands.
|
|
877
993
|
* @param {import("./config.js").RollbridgeConfig} config - Loaded config.
|
|
878
994
|
* @param {number} timeoutMs - Timeout in milliseconds.
|
|
995
|
+
* @param {{candidate?: import("node:child_process").ChildProcess, configDigest?: string, runtime?: import("./daemon-runtime.js").DaemonRuntimeIdentity}} [expected] - Required authority and exact spawned candidate.
|
|
879
996
|
* @returns {Promise<Record<string, import("./json.js").JsonValue>>} Daemon status response.
|
|
880
997
|
*/
|
|
881
|
-
async function waitForDaemonStatus(config, timeoutMs) {
|
|
998
|
+
async function waitForDaemonStatus(config, timeoutMs, expected = {}) {
|
|
882
999
|
const deadline = Date.now() + timeoutMs
|
|
883
1000
|
let lastError = /** @type {Error | undefined} */ (undefined)
|
|
1001
|
+
let candidateExit = expected.candidate && (expected.candidate.exitCode !== null || expected.candidate.signalCode !== null)
|
|
1002
|
+
? {code: expected.candidate.exitCode, signal: expected.candidate.signalCode}
|
|
1003
|
+
: undefined
|
|
1004
|
+
let wakeDelay = /** @type {(() => void) | undefined} */ (undefined)
|
|
1005
|
+
const onCandidateExit = (/** @type {number | null} */ code, /** @type {DaemonExitSignal} */ signal) => {
|
|
1006
|
+
candidateExit = {code, signal}
|
|
1007
|
+
wakeDelay?.()
|
|
1008
|
+
}
|
|
1009
|
+
|
|
1010
|
+
expected.candidate?.once("exit", onCandidateExit)
|
|
1011
|
+
|
|
1012
|
+
try {
|
|
1013
|
+
while (Date.now() < deadline) {
|
|
1014
|
+
let status
|
|
1015
|
+
|
|
1016
|
+
try {
|
|
1017
|
+
status = await daemonStatus(config)
|
|
1018
|
+
} catch (error) {
|
|
1019
|
+
lastError = error instanceof Error ? error : new Error(String(error))
|
|
1020
|
+
}
|
|
884
1021
|
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
1022
|
+
if (status) {
|
|
1023
|
+
const statusPid = typeof status.daemonPid === "number" ? status.daemonPid : undefined
|
|
1024
|
+
const candidateResolved = !expected.candidate || candidateExit || statusPid === expected.candidate.pid
|
|
888
1025
|
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
1026
|
+
if (expected.runtime && !compatibleDaemonRuntime(status, expected.runtime)) {
|
|
1027
|
+
if (!expected.configDigest) assertCompatibleDaemonRuntime(status, expected.runtime)
|
|
1028
|
+
} else if (candidateResolved && (!expected.configDigest || (status.ownerRecovery && typeof status.ownerRecovery === "object" && !Array.isArray(status.ownerRecovery) && status.ownerRecovery.configDigest === expected.configDigest))) {
|
|
1029
|
+
return status
|
|
1030
|
+
}
|
|
1031
|
+
}
|
|
1032
|
+
|
|
1033
|
+
if (candidateExit && expected.candidate) throw candidateExitError(expected.candidate, candidateExit)
|
|
1034
|
+
await new Promise((resolve) => {
|
|
1035
|
+
const timer = setTimeout(finish, 100)
|
|
1036
|
+
/** Completes one readiness interval or candidate-exit wake-up. */
|
|
1037
|
+
function finish() {
|
|
1038
|
+
clearTimeout(timer)
|
|
1039
|
+
wakeDelay = undefined
|
|
1040
|
+
resolve(undefined)
|
|
1041
|
+
}
|
|
1042
|
+
|
|
1043
|
+
wakeDelay = finish
|
|
1044
|
+
})
|
|
892
1045
|
}
|
|
893
1046
|
|
|
894
|
-
|
|
895
|
-
}
|
|
1047
|
+
const detail = lastError ? ` Last error: ${lastError.message}` : ""
|
|
896
1048
|
|
|
897
|
-
|
|
1049
|
+
throw new Error(`Rollbridge daemon did not become ready within ${timeoutMs}ms.${detail}`)
|
|
1050
|
+
} finally {
|
|
1051
|
+
expected.candidate?.off("exit", onCandidateExit)
|
|
1052
|
+
wakeDelay = undefined
|
|
1053
|
+
}
|
|
1054
|
+
}
|
|
898
1055
|
|
|
899
|
-
|
|
1056
|
+
/**
|
|
1057
|
+
* @param {import("node:child_process").ChildProcess} candidate - Exact spawned daemon candidate.
|
|
1058
|
+
* @param {{code: number | null, signal: DaemonExitSignal}} exit - Exact exit status.
|
|
1059
|
+
* @returns {Error} Actionable pre-readiness failure.
|
|
1060
|
+
*/
|
|
1061
|
+
function candidateExitError(candidate, exit) {
|
|
1062
|
+
return new Error(
|
|
1063
|
+
`Rollbridge daemon candidate ${candidate.pid ?? "unknown"} exited before readiness ` +
|
|
1064
|
+
`(code ${exit.code ?? "none"}, signal ${exit.signal ?? "none"}). Spawned argv: ${JSON.stringify(candidate.spawnargs)}`
|
|
1065
|
+
)
|
|
900
1066
|
}
|
|
901
1067
|
|
|
902
1068
|
/**
|
|
@@ -933,11 +1099,7 @@ function defaultDaemonRuntimePath(config) {
|
|
|
933
1099
|
* @returns {void}
|
|
934
1100
|
*/
|
|
935
1101
|
function assertCompatibleDaemonRuntime(status, expected) {
|
|
936
|
-
|
|
937
|
-
const compatible = runtime && typeof runtime === "object" && !Array.isArray(runtime) &&
|
|
938
|
-
runtime.format === expected.format && runtime.version === expected.version && runtime.digest === expected.digest
|
|
939
|
-
|
|
940
|
-
if (compatible) return
|
|
1102
|
+
if (compatibleDaemonRuntime(status, expected)) return
|
|
941
1103
|
|
|
942
1104
|
throw new Error(
|
|
943
1105
|
"The running Rollbridge daemon has a legacy or mismatched runtime. " +
|
|
@@ -945,6 +1107,17 @@ function assertCompatibleDaemonRuntime(status, expected) {
|
|
|
945
1107
|
)
|
|
946
1108
|
}
|
|
947
1109
|
|
|
1110
|
+
/**
|
|
1111
|
+
* @param {Record<string, import("./json.js").JsonValue>} status - Daemon status.
|
|
1112
|
+
* @param {import("./daemon-runtime.js").DaemonRuntimeIdentity} expected - Runtime.
|
|
1113
|
+
* @returns {boolean} Whether the runtime authorities match.
|
|
1114
|
+
*/
|
|
1115
|
+
function compatibleDaemonRuntime(status, expected) {
|
|
1116
|
+
const runtime = status.daemonRuntime
|
|
1117
|
+
|
|
1118
|
+
return Boolean(runtime && typeof runtime === "object" && !Array.isArray(runtime) && runtime.format === expected.format && runtime.version === expected.version && runtime.digest === expected.digest)
|
|
1119
|
+
}
|
|
1120
|
+
|
|
948
1121
|
/**
|
|
949
1122
|
* @param {string | undefined} value - Raw timeout value.
|
|
950
1123
|
* @returns {number} Timeout in milliseconds.
|
package/src/config.js
CHANGED
|
@@ -23,7 +23,8 @@ import {pathToFileURL} from "node:url"
|
|
|
23
23
|
* @typedef {{includes: string[], name: string}} LegacyTakeoverProcessConfig
|
|
24
24
|
* @typedef {{forceStopTimeoutMs: number, processes: LegacyTakeoverProcessConfig[], screens: string[]}} LegacyTakeoverConfig
|
|
25
25
|
* @typedef {{keep: number, maxAgeMs: number}} ReleaseRetentionConfig
|
|
26
|
-
* @typedef {{
|
|
26
|
+
* @typedef {{reconnectGraceMs: number}} OwnerRecoveryConfig
|
|
27
|
+
* @typedef {{application: string, control: ControlConfig, legacyTakeover?: LegacyTakeoverConfig, ownerRecovery?: OwnerRecoveryConfig, processes: ProcessConfig[], proxy: ProxyConfig, releaseRetention: ReleaseRetentionConfig, statePath?: string}} RollbridgeConfig
|
|
27
28
|
* @typedef {{fix: string, message: string}} ConfigIssue
|
|
28
29
|
*/
|
|
29
30
|
|
|
@@ -150,10 +151,15 @@ export function validateConfig(rawConfig, configPath = process.cwd()) {
|
|
|
150
151
|
const legacyTakeover = normalizeLegacyTakeover(source.legacyTakeover, proxy, issues)
|
|
151
152
|
const releaseRetention = normalizeReleaseRetention(objectAt(source.releaseRetention, "releaseRetention", issues, {}), issues)
|
|
152
153
|
const statePath = source.statePath === undefined || source.statePath === null ? undefined : normalizeString(source.statePath, "statePath", issues)
|
|
154
|
+
const ownerRecoverySource = source.ownerRecovery === undefined || source.ownerRecovery === null ? undefined : objectAt(source.ownerRecovery, "ownerRecovery", issues)
|
|
155
|
+
const reconnectGraceMs = ownerRecoverySource ? normalizeNumber(ownerRecoverySource.reconnectGraceMs, "ownerRecovery.reconnectGraceMs", issues, {default: 30000}) : 30000
|
|
156
|
+
const ownerRecovery = ownerRecoverySource ? {reconnectGraceMs: nonNegativeOrDefault(reconnectGraceMs, "ownerRecovery.reconnectGraceMs", issues, 30000, true)} : undefined
|
|
157
|
+
|
|
158
|
+
if (ownerRecovery && !statePath) issues.push({fix: "Configure statePath when ownerRecovery is enabled.", message: "ownerRecovery requires statePath"})
|
|
153
159
|
|
|
154
160
|
validateProcessSet(processes, issues)
|
|
155
161
|
|
|
156
|
-
return {config: {application, control, legacyTakeover, processes, proxy, releaseRetention, statePath}, issues}
|
|
162
|
+
return {config: {application, control, legacyTakeover, ownerRecovery, processes, proxy, releaseRetention, statePath}, issues}
|
|
157
163
|
}
|
|
158
164
|
|
|
159
165
|
/**
|
package/src/control-client.js
CHANGED
|
@@ -27,7 +27,15 @@ export async function sendControlCommand({command, path}) {
|
|
|
27
27
|
if (newlineIndex < 0) return
|
|
28
28
|
|
|
29
29
|
const line = buffer.slice(0, newlineIndex)
|
|
30
|
-
|
|
30
|
+
let response
|
|
31
|
+
|
|
32
|
+
try {
|
|
33
|
+
response = JSON.parse(line)
|
|
34
|
+
} catch (error) {
|
|
35
|
+
socket.destroy()
|
|
36
|
+
reject(new Error(`Invalid Rollbridge control response from ${path}`, {cause: error}))
|
|
37
|
+
return
|
|
38
|
+
}
|
|
31
39
|
|
|
32
40
|
socket.end()
|
|
33
41
|
|
|
@@ -40,3 +48,112 @@ export async function sendControlCommand({command, path}) {
|
|
|
40
48
|
socket.write(`${JSON.stringify(command)}\n`)
|
|
41
49
|
})
|
|
42
50
|
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Opens one incumbent control connection that survives listener unlink during handoff.
|
|
54
|
+
* @param {string} path - Control socket path.
|
|
55
|
+
* @returns {Promise<ControlSession>} Connected session.
|
|
56
|
+
*/
|
|
57
|
+
export async function openControlSession(path) {
|
|
58
|
+
const session = new ControlSession(path)
|
|
59
|
+
|
|
60
|
+
await session.connect()
|
|
61
|
+
return session
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
class ControlSession {
|
|
65
|
+
/** @param {string} path - Control socket path. */
|
|
66
|
+
constructor(path) {
|
|
67
|
+
this.path = path
|
|
68
|
+
this.socket = /** @type {net.Socket | undefined} */ (undefined)
|
|
69
|
+
this.buffer = ""
|
|
70
|
+
this.pending = /** @type {{reject: (error: Error) => void, resolve: (value: Record<string, JsonValue>) => void} | undefined} */ (undefined)
|
|
71
|
+
this.eventHandlers = /** @type {((event: Record<string, JsonValue>) => void)[]} */ ([])
|
|
72
|
+
this.closePromise = /** @type {Promise<void> | undefined} */ (undefined)
|
|
73
|
+
this.resolveClose = /** @type {(() => void) | undefined} */ (undefined)
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** Connects before the incumbent stops accepting new control clients. */
|
|
77
|
+
async connect() {
|
|
78
|
+
const socket = net.createConnection(this.path)
|
|
79
|
+
|
|
80
|
+
socket.setEncoding("utf8")
|
|
81
|
+
await new Promise((resolve, reject) => {
|
|
82
|
+
socket.once("connect", resolve)
|
|
83
|
+
socket.once("error", reject)
|
|
84
|
+
})
|
|
85
|
+
socket.on("data", (chunk) => this.onData(String(chunk)))
|
|
86
|
+
this.closePromise = new Promise((resolve) => { this.resolveClose = () => resolve(undefined) })
|
|
87
|
+
socket.once("close", () => {
|
|
88
|
+
this.pending?.reject(new Error(`Rollbridge control connection ${this.path} closed before its response`))
|
|
89
|
+
this.pending = undefined
|
|
90
|
+
this.resolveClose?.()
|
|
91
|
+
})
|
|
92
|
+
this.socket = socket
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* @param {Record<string, JsonValue>} command - Command payload.
|
|
97
|
+
* @returns {Promise<Record<string, JsonValue>>} Response payload.
|
|
98
|
+
*/
|
|
99
|
+
async request(command) {
|
|
100
|
+
if (!this.socket || this.socket.destroyed) throw new Error(`Rollbridge control connection ${this.path} is not connected`)
|
|
101
|
+
if (this.pending) throw new Error("Rollbridge control session permits one in-flight command")
|
|
102
|
+
const response = new Promise((resolve, reject) => { this.pending = {reject, resolve} })
|
|
103
|
+
|
|
104
|
+
this.socket.write(`${JSON.stringify(command)}\n`)
|
|
105
|
+
return await response
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/** Closes the persistent handoff connection. */
|
|
109
|
+
close() {
|
|
110
|
+
this.socket?.destroy()
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** Waits for the kernel to close the incumbent control connection. */
|
|
114
|
+
async closed() {
|
|
115
|
+
if (!this.closePromise) throw new Error(`Rollbridge control connection ${this.path} was not opened`)
|
|
116
|
+
await this.closePromise
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* Subscribes to authenticated incumbent handoff events.
|
|
121
|
+
* @param {(event: Record<string, JsonValue>) => void} handler - Event handler.
|
|
122
|
+
*/
|
|
123
|
+
onEvent(handler) {
|
|
124
|
+
this.eventHandlers.push(handler)
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** @param {string} chunk - Protocol bytes. */
|
|
128
|
+
onData(chunk) {
|
|
129
|
+
this.buffer += chunk
|
|
130
|
+
let newlineIndex = this.buffer.indexOf("\n")
|
|
131
|
+
|
|
132
|
+
while (newlineIndex >= 0) {
|
|
133
|
+
const line = this.buffer.slice(0, newlineIndex)
|
|
134
|
+
this.buffer = this.buffer.slice(newlineIndex + 1)
|
|
135
|
+
|
|
136
|
+
try {
|
|
137
|
+
const response = JSON.parse(line)
|
|
138
|
+
|
|
139
|
+
if (response.event) {
|
|
140
|
+
for (const handler of this.eventHandlers) handler(response)
|
|
141
|
+
} else {
|
|
142
|
+
const pending = this.pending
|
|
143
|
+
|
|
144
|
+
this.pending = undefined
|
|
145
|
+
if (pending) {
|
|
146
|
+
if (response.status === "error") pending.reject(new Error(String(response.error || "Unknown Rollbridge error")))
|
|
147
|
+
else pending.resolve(response)
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
} catch (error) {
|
|
151
|
+
const pending = this.pending
|
|
152
|
+
|
|
153
|
+
this.pending = undefined
|
|
154
|
+
pending?.reject(new Error(`Invalid Rollbridge control response from ${this.path}`, {cause: error}))
|
|
155
|
+
}
|
|
156
|
+
newlineIndex = this.buffer.indexOf("\n")
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
}
|