rollbridge 0.1.39 → 0.1.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -4
- package/changelog.d/20260830-guardian-retired-replacement-process-key.md +12 -1
- package/changelog.d/20260830-release-generation-activation-lifecycle.md +16 -2
- package/changelog.d/20260830055159-guardian-daemon-restart.md +2 -0
- package/docs/cli.md +23 -9
- package/docs/config.md +37 -13
- package/docs/logging.md +4 -3
- package/docs/troubleshooting.md +7 -5
- package/examples/tensorbuzz.com.js +5 -2
- package/package.json +1 -1
- package/src/cli.js +118 -24
- package/src/config.js +35 -8
- package/src/daemon.js +822 -152
- package/src/guardian-client.js +121 -16
- package/src/managed-process.js +117 -15
- package/src/process-guardian.js +734 -43
- package/src/release-group.js +45 -7
- package/test/completion.test.js +4 -2
- package/test/config-examples.test.js +1 -0
- package/test/config-validation.test.js +22 -0
- package/test/fixtures/guardian-recovery-owner.js +86 -0
- package/test/fixtures/pre-split3-process-guardian.js +14 -0
- package/test/guardian-client.test.js +1420 -62
- package/test/managed-process.test.js +163 -7
- package/test/owner-recovery.test.js +575 -73
- package/test/owner-replacement.test.js +525 -28
- package/test/release-group.test.js +19 -2
- package/test/release-runtime-retention.test.js +121 -4
- package/test/rollbridge.test.js +263 -13
- package/test/support/process.js +41 -0
package/src/process-guardian.js
CHANGED
|
@@ -1,32 +1,48 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
|
+
import fsSync from "node:fs"
|
|
3
4
|
import fs from "node:fs/promises"
|
|
4
5
|
import net from "node:net"
|
|
5
6
|
import crypto from "node:crypto"
|
|
7
|
+
import {spawn} from "node:child_process"
|
|
6
8
|
import {isDeepStrictEqual} from "node:util"
|
|
9
|
+
import path from "node:path"
|
|
7
10
|
import GuardianClient from "./guardian-client.js"
|
|
8
11
|
import ManagedProcess from "./managed-process.js"
|
|
12
|
+
import {writeState} from "./state-store.js"
|
|
13
|
+
|
|
14
|
+
const OWNER_RESTART_RETRY_MS = 1000
|
|
15
|
+
|
|
16
|
+
/** @typedef {import("node:child_process").ChildProcess["signalCode"]} ChildExitSignal */
|
|
9
17
|
|
|
10
18
|
/**
|
|
11
19
|
* @typedef {object} GuardianRequest
|
|
12
20
|
* @property {string} command - Operation name.
|
|
21
|
+
* @property {{http?: number, websocket?: number}} [connections] - Retired listener counts.
|
|
13
22
|
* @property {ConstructorParameters<typeof ManagedProcess>[0]} [definition] - Managed process definition.
|
|
14
23
|
* @property {number} [graceMs] - Owner reconnection grace.
|
|
15
24
|
* @property {number} id - Request id.
|
|
16
25
|
* @property {string} [key] - Stable process key.
|
|
26
|
+
* @property {boolean} [localSource] - Whether the retired daemon physically owns the published source.
|
|
17
27
|
* @property {{timeoutMs?: number}} [options] - Stop options.
|
|
18
28
|
* @property {string} [previousProvenance] - Expected current provenance.
|
|
19
29
|
* @property {string} [provenance] - New or registered provenance.
|
|
30
|
+
* @property {string} [releaseId] - Retained release identity.
|
|
31
|
+
* @property {string} [sourceId] - Stable retired-listener source identity.
|
|
20
32
|
* @property {import("./json.js").JsonValue} [ownerState] - Private owner transfer state.
|
|
21
33
|
* @property {string} [replacementId] - Prepared replacement transaction id.
|
|
22
34
|
* @property {string} [mutationId] - Owner mutation lease id.
|
|
23
35
|
* @property {string} [operation] - Owner mutation diagnostic name.
|
|
36
|
+
* @property {number} [ownerPid] - Exact claimant process PID.
|
|
37
|
+
* @property {import("./json.js").JsonValue} [recoverySnapshot] - Public bridge recovery snapshot.
|
|
38
|
+
* @property {string} [statePath] - Exact public recovery state path.
|
|
24
39
|
* @property {import("./json.js").JsonValue} [authority] - Expected current authority.
|
|
25
40
|
* @property {import("./json.js").JsonValue} [nextAuthority] - Requested replacement authority.
|
|
26
41
|
* @property {import("./managed-process.js").ManagedProcessStartReason} [reason] - Start reason.
|
|
27
42
|
* @property {import("./managed-process.js").LifecycleRole} [lifecycleRole] - Desired generation role restored during start.
|
|
28
43
|
* @property {string} token - Authentication token.
|
|
29
44
|
*/
|
|
45
|
+
/** @typedef {{listenerStateComplete: boolean, localSources: Map<string, Map<string, {http: number, websocket: number}>>, replacementId: string, successor: net.Socket}} RetiredListenerTransaction */
|
|
30
46
|
|
|
31
47
|
const [socketPath] = process.argv.slice(2)
|
|
32
48
|
|
|
@@ -51,10 +67,24 @@ if (legacyGuardian) await legacyGuardian.connect()
|
|
|
51
67
|
const processes = new Map()
|
|
52
68
|
/** @type {Set<net.Socket>} */
|
|
53
69
|
const clients = new Set()
|
|
70
|
+
/** @type {Set<net.Socket>} */
|
|
71
|
+
const backpressuredClients = new Set()
|
|
72
|
+
/** @type {Map<net.Socket, Map<string, string>>} */
|
|
73
|
+
const pendingStatusEvents = new Map()
|
|
74
|
+
/** @type {Map<net.Socket, number>} */
|
|
75
|
+
const ownerUpdatesInFlight = new Map()
|
|
76
|
+
/** @type {Map<net.Socket, RetiredListenerTransaction>} */
|
|
77
|
+
const retiredListenerClients = new Map()
|
|
78
|
+
/** @type {Set<RetiredListenerTransaction>} */
|
|
79
|
+
const disconnectedOwnerSuccessors = new Set()
|
|
54
80
|
/** @type {net.Socket | undefined} */
|
|
55
81
|
let ownerClient
|
|
82
|
+
/** @type {number | undefined} */
|
|
83
|
+
let ownerClientPid
|
|
56
84
|
/** @type {net.Socket | undefined} */
|
|
57
85
|
let replacementClient
|
|
86
|
+
/** @type {number | undefined} */
|
|
87
|
+
let replacementOwnerPid
|
|
58
88
|
/** @type {string | undefined} */
|
|
59
89
|
let replacementId
|
|
60
90
|
/** @type {string | undefined} */
|
|
@@ -77,15 +107,29 @@ let ownerMutationId
|
|
|
77
107
|
let retiringClient
|
|
78
108
|
/** @type {string | undefined} */
|
|
79
109
|
let retiringReplacementId
|
|
110
|
+
let retiringListenerReady = false
|
|
111
|
+
let retirementFailed = false
|
|
80
112
|
/** @type {Promise<Error | undefined> | undefined} */
|
|
81
113
|
let legacyOwnerClaim
|
|
114
|
+
/** @type {import("./json.js").JsonValue | undefined} */
|
|
115
|
+
let legacyRecoverySnapshot
|
|
116
|
+
/** @type {string | undefined} */
|
|
117
|
+
let legacyRecoveryStatePath
|
|
82
118
|
let shuttingDown = false
|
|
83
119
|
/** @type {net.Socket | undefined} */
|
|
84
120
|
let shutdownClient
|
|
85
121
|
let shutdownFinalizing = false
|
|
86
122
|
/** @type {Promise<void> | undefined} */
|
|
87
123
|
let serverClosed
|
|
88
|
-
/** @type {
|
|
124
|
+
/** @type {import("node:child_process").ChildProcess | undefined} */
|
|
125
|
+
let ownerRestartChild
|
|
126
|
+
/** @type {ReturnType<typeof setTimeout> | undefined} */
|
|
127
|
+
let ownerRestartTimer
|
|
128
|
+
/** @type {ReturnType<typeof setTimeout> | undefined} */
|
|
129
|
+
let ownerRestartStartupTimer
|
|
130
|
+
/** @type {number | undefined} */
|
|
131
|
+
let ownerRestartRetryDelayMs
|
|
132
|
+
/** @type {{authority: import("./json.js").JsonValue, ownerPid: number | undefined, reject: (error: Error) => void, resolve: (value: {claimed: boolean}) => void, socket: net.Socket, timer: ReturnType<typeof setTimeout>}[]} */
|
|
89
133
|
const claimWaiters = []
|
|
90
134
|
|
|
91
135
|
const server = net.createServer((socket) => {
|
|
@@ -94,10 +138,15 @@ const server = net.createServer((socket) => {
|
|
|
94
138
|
// An abruptly killed daemon can reset its private guardian connection. Keep the
|
|
95
139
|
// durable guardian alive; the close handler below releases only that owner claim.
|
|
96
140
|
socket.on("error", () => socket.destroy())
|
|
141
|
+
socket.on("drain", () => flushStatusEvents(socket))
|
|
97
142
|
let buffer = ""
|
|
98
143
|
|
|
99
144
|
socket.once("close", () => {
|
|
145
|
+
const controlLessRetirement = retiredListenerClients.get(socket)
|
|
146
|
+
|
|
100
147
|
clients.delete(socket)
|
|
148
|
+
backpressuredClients.delete(socket)
|
|
149
|
+
pendingStatusEvents.delete(socket)
|
|
101
150
|
const waiterIndex = claimWaiters.findIndex((waiter) => waiter.socket === socket)
|
|
102
151
|
|
|
103
152
|
if (waiterIndex >= 0) {
|
|
@@ -105,21 +154,46 @@ const server = net.createServer((socket) => {
|
|
|
105
154
|
clearTimeout(waiter.timer)
|
|
106
155
|
waiter.reject(new Error("Owner claimant disconnected"))
|
|
107
156
|
}
|
|
108
|
-
if (
|
|
109
|
-
ownerClient = undefined
|
|
110
|
-
if (replacementClient && replacementOwnerState) commitReplacement()
|
|
111
|
-
else {
|
|
112
|
-
if (replacementClient) abortReplacement("Committed owner disconnected before replacement candidate was ready")
|
|
113
|
-
if (!shuttingDown) grantNextOwner()
|
|
114
|
-
}
|
|
115
|
-
}
|
|
116
|
-
if (ownerMutationClient === socket) {
|
|
157
|
+
if (ownerMutationClient === socket && ownerClient !== socket) {
|
|
117
158
|
ownerMutationClient = undefined
|
|
118
159
|
ownerMutationId = undefined
|
|
119
160
|
}
|
|
120
|
-
if (
|
|
161
|
+
if (controlLessRetirement) {
|
|
162
|
+
const preCommitHandoff = replacementClient && ownerClient === socket && retiringReplacementId === replacementId
|
|
163
|
+
|
|
164
|
+
if (preCommitHandoff) {
|
|
165
|
+
publishClosedLocalSources(controlLessRetirement)
|
|
166
|
+
if (!controlLessRetirement.successor.destroyed) {
|
|
167
|
+
controlLessRetirement.successor.write(`${JSON.stringify({event: "replacement-retirement-failed", reason: "Incumbent listener disconnected during the prepared handoff", replacementId: controlLessRetirement.replacementId})}\n`)
|
|
168
|
+
}
|
|
169
|
+
abortReplacement("Incumbent listener disconnected during the prepared handoff", false)
|
|
170
|
+
} else if (retiringClient === socket && !controlLessRetirement.listenerStateComplete) {
|
|
171
|
+
retirementFailed = true
|
|
172
|
+
retiringClient = undefined
|
|
173
|
+
retiringReplacementId = undefined
|
|
174
|
+
if (ownerClient && !ownerClient.destroyed) {
|
|
175
|
+
ownerClient.write(`${JSON.stringify({event: "replacement-retirement-failed", reason: "Retired listener disconnected before publishing complete connection state", replacementId: controlLessRetirement.replacementId})}\n`)
|
|
176
|
+
}
|
|
177
|
+
} else if (controlLessRetirement.listenerStateComplete) {
|
|
178
|
+
publishClosedLocalSources(controlLessRetirement)
|
|
179
|
+
if (retiringClient === socket) finalizeReplacementRetirement()
|
|
180
|
+
}
|
|
181
|
+
} else if (retiringClient === socket) finalizeReplacementRetirement()
|
|
182
|
+
if (controlLessRetirement && retiredListenerClients.has(socket)) {
|
|
183
|
+
for (const transaction of retiredListenerClients.values()) {
|
|
184
|
+
if (transaction.successor === socket) transaction.successor = controlLessRetirement.successor
|
|
185
|
+
}
|
|
186
|
+
disconnectedOwnerSuccessors.delete(controlLessRetirement)
|
|
187
|
+
retiredListenerClients.delete(socket)
|
|
188
|
+
}
|
|
189
|
+
if (ownerClient === socket && !ownerUpdatesInFlight.has(socket)) releaseClosedOwner(socket)
|
|
121
190
|
if (replacementClient === socket) {
|
|
122
191
|
abortReplacement("Replacement candidate disconnected before commit")
|
|
192
|
+
if (legacyGuardian && !legacyOwnerClaim && !ownerClient && !committedReplacementId) {
|
|
193
|
+
legacyGuardian.disconnect()
|
|
194
|
+
shuttingDown = true
|
|
195
|
+
beginSuccessfulShutdown(socket)
|
|
196
|
+
}
|
|
123
197
|
}
|
|
124
198
|
if (shutdownClient === socket) void finishShutdown()
|
|
125
199
|
})
|
|
@@ -158,15 +232,22 @@ server.listen(socketPath, async () => {
|
|
|
158
232
|
async function handleLine(socket, line) {
|
|
159
233
|
/** @type {GuardianRequest | undefined} */
|
|
160
234
|
let request
|
|
235
|
+
let ownerUpdate = false
|
|
161
236
|
|
|
162
237
|
try {
|
|
163
238
|
request = /** @type {GuardianRequest} */ (JSON.parse(line))
|
|
164
239
|
if (request.token !== token) throw new Error("Guardian authentication failed")
|
|
240
|
+
if (ownerClient === socket && request.command === "update") {
|
|
241
|
+
ownerUpdate = true
|
|
242
|
+
ownerUpdatesInFlight.set(socket, (ownerUpdatesInFlight.get(socket) ?? 0) + 1)
|
|
243
|
+
}
|
|
165
244
|
const result = await execute(request, socket)
|
|
166
245
|
|
|
167
246
|
socket.write(`${JSON.stringify({id: request.id, result})}\n`)
|
|
168
247
|
} catch (error) {
|
|
169
248
|
if (!socket.destroyed) socket.write(`${JSON.stringify({error: error instanceof Error ? error.message : String(error), id: request?.id})}\n`)
|
|
249
|
+
} finally {
|
|
250
|
+
if (ownerUpdate) completeOwnerUpdate(socket)
|
|
170
251
|
}
|
|
171
252
|
}
|
|
172
253
|
|
|
@@ -178,16 +259,24 @@ async function handleLine(socket, line) {
|
|
|
178
259
|
async function execute(request, socket) {
|
|
179
260
|
if (shuttingDown) throw new Error("Process guardian is shutting down")
|
|
180
261
|
|
|
262
|
+
if (request.command === "capabilities") return {daemonRecovery: 1, generationReactivation: 1}
|
|
263
|
+
|
|
181
264
|
if (request.command === "owner-replacement-capabilities") {
|
|
182
265
|
return {commands: ["commit-retired-owner-replacement"], protocol: "owner-replacement", version: 1}
|
|
183
266
|
}
|
|
184
267
|
|
|
185
268
|
if (request.command === "claim-owner") {
|
|
269
|
+
if (retirementFailed) throw new Error("Guardian owner replacement is fenced after incomplete retired-listener state transfer")
|
|
270
|
+
if (ownerState !== undefined && !isDeepStrictEqual(request.authority, ownerAuthority(ownerState))) {
|
|
271
|
+
const waitsForPreparedAuthority = Boolean(ownerClient && replacementOwnerState && isDeepStrictEqual(request.authority, replacementAuthority))
|
|
272
|
+
|
|
273
|
+
if (!waitsForPreparedAuthority) throw new Error("Owner recovery authority does not match the guardian's committed authority")
|
|
274
|
+
}
|
|
186
275
|
if (!ownerClient) {
|
|
187
|
-
|
|
188
|
-
throw new Error("Owner recovery authority does not match the guardian's committed authority")
|
|
189
|
-
}
|
|
276
|
+
acceptOwnerClaim(request.ownerPid)
|
|
190
277
|
ownerClient = socket
|
|
278
|
+
ownerClientPid = request.ownerPid
|
|
279
|
+
reattachRetiredListenerSuccessors(socket)
|
|
191
280
|
return {claimed: true}
|
|
192
281
|
}
|
|
193
282
|
if (ownerClient === socket) return {claimed: true}
|
|
@@ -199,7 +288,7 @@ async function execute(request, socket) {
|
|
|
199
288
|
reject(new Error("Durable owner is already claimed by another matching daemon"))
|
|
200
289
|
}, request.graceMs ?? 30000)
|
|
201
290
|
|
|
202
|
-
claimWaiters.push({reject, resolve, socket, timer})
|
|
291
|
+
claimWaiters.push({authority: request.authority ?? null, ownerPid: request.ownerPid, reject, resolve, socket, timer})
|
|
203
292
|
})
|
|
204
293
|
}
|
|
205
294
|
|
|
@@ -208,7 +297,10 @@ async function execute(request, socket) {
|
|
|
208
297
|
if (replacementClient) throw new Error("Committed owner cannot retire while an owner replacement is prepared")
|
|
209
298
|
for (const entry of processes.values()) entry.desired = false
|
|
210
299
|
void Promise.allSettled([...processes.values()].map((entry) => entry.process.stop()))
|
|
300
|
+
clearOwnerLocalListenerSource()
|
|
301
|
+
rememberDisconnectedOwnerSuccessors(socket)
|
|
211
302
|
ownerClient = undefined
|
|
303
|
+
ownerClientPid = undefined
|
|
212
304
|
ownerMutationClient = undefined
|
|
213
305
|
ownerMutationId = undefined
|
|
214
306
|
ownerRevision += 1
|
|
@@ -216,6 +308,34 @@ async function execute(request, socket) {
|
|
|
216
308
|
return {retired: true}
|
|
217
309
|
}
|
|
218
310
|
|
|
311
|
+
if (request.command === "owner-ready") {
|
|
312
|
+
requireOwner(socket, request.command)
|
|
313
|
+
if (!request.ownerPid || request.ownerPid !== ownerClientPid) throw new Error("Guardian owner readiness PID does not match the claimed owner")
|
|
314
|
+
const recovery = ownerRecoveryDefinition()
|
|
315
|
+
const pidPath = recovery?.command.pidPath
|
|
316
|
+
|
|
317
|
+
if (ownerRestartChild && ownerRestartChild.pid !== request.ownerPid) throw new Error("Guardian owner readiness does not match the tracked recovery child")
|
|
318
|
+
if (pidPath) {
|
|
319
|
+
let temporaryDirectory
|
|
320
|
+
|
|
321
|
+
fsSync.mkdirSync(path.dirname(pidPath), {recursive: true})
|
|
322
|
+
try {
|
|
323
|
+
temporaryDirectory = fsSync.mkdtempSync(path.join(path.dirname(pidPath), `.${path.basename(pidPath)}.`))
|
|
324
|
+
const temporaryPath = path.join(temporaryDirectory, "pid")
|
|
325
|
+
|
|
326
|
+
fsSync.writeFileSync(temporaryPath, `${request.ownerPid}\n`, {flag: "wx"})
|
|
327
|
+
fsSync.renameSync(temporaryPath, pidPath)
|
|
328
|
+
} finally {
|
|
329
|
+
if (temporaryDirectory) fsSync.rmSync(temporaryDirectory, {force: true, recursive: true})
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
if (ownerRestartChild) {
|
|
333
|
+
clearOwnerRestartTracking(ownerRestartChild)
|
|
334
|
+
}
|
|
335
|
+
ownerRestartRetryDelayMs = undefined
|
|
336
|
+
return {ready: true}
|
|
337
|
+
}
|
|
338
|
+
|
|
219
339
|
if (request.command === "abandon-legacy-upgrade") {
|
|
220
340
|
if (!legacyGuardian) throw new Error("Guardian is not a legacy upgrade coordinator")
|
|
221
341
|
if (ownerClient || committedReplacementId) throw new Error("Committed guardian authority cannot abandon its legacy backend")
|
|
@@ -231,7 +351,13 @@ async function execute(request, socket) {
|
|
|
231
351
|
if (!legacyGuardian) throw new Error("Guardian is not a legacy upgrade coordinator")
|
|
232
352
|
requireReplacement(socket, request)
|
|
233
353
|
if (legacyOwnerClaim) throw new Error("Legacy guardian owner claim is already pending")
|
|
234
|
-
|
|
354
|
+
if (replacementAuthority === undefined) throw new Error("Legacy guardian owner claim is missing replacement authority")
|
|
355
|
+
if (!request.statePath || !path.isAbsolute(request.statePath)) throw new Error("Legacy guardian owner claim requires an absolute state path")
|
|
356
|
+
if (request.statePath !== ownerStatePath(ownerState)) throw new Error("Legacy guardian owner claim state path does not match the committed owner config")
|
|
357
|
+
assertBridgeRecoverySnapshot(request.recoverySnapshot)
|
|
358
|
+
legacyRecoveryStatePath = request.statePath
|
|
359
|
+
legacyRecoverySnapshot = request.recoverySnapshot
|
|
360
|
+
legacyOwnerClaim = legacyGuardian.claimOwner(request.graceMs ?? 30000, replacementAuthority).then(
|
|
235
361
|
() => undefined,
|
|
236
362
|
(error) => error instanceof Error ? error : new Error(String(error))
|
|
237
363
|
)
|
|
@@ -244,15 +370,17 @@ async function execute(request, socket) {
|
|
|
244
370
|
const claimError = await legacyOwnerClaim
|
|
245
371
|
|
|
246
372
|
if (claimError) throw claimError
|
|
373
|
+
if (!legacyRecoveryStatePath || legacyRecoverySnapshot === undefined) throw new Error("Legacy guardian owner claim is missing durable recovery state")
|
|
374
|
+
await writeState(legacyRecoveryStatePath, legacyRecoverySnapshot)
|
|
247
375
|
return {claimed: true}
|
|
248
376
|
}
|
|
249
377
|
|
|
250
378
|
if (request.command === "publish-owner-state") {
|
|
251
379
|
requireOwner(socket, request.command)
|
|
252
380
|
if (request.ownerState === undefined) throw new Error("Guardian owner publication requires ownerState")
|
|
253
|
-
ownerState = request.ownerState
|
|
381
|
+
ownerState = mergeOwnerConnectionState(ownerState, request.ownerState)
|
|
254
382
|
ownerRevision += 1
|
|
255
|
-
committedReplacementId = undefined
|
|
383
|
+
if (!retiringClient) committedReplacementId = undefined
|
|
256
384
|
return {published: true}
|
|
257
385
|
}
|
|
258
386
|
|
|
@@ -263,7 +391,7 @@ async function execute(request, socket) {
|
|
|
263
391
|
}
|
|
264
392
|
|
|
265
393
|
if (request.command === "replacement-status") {
|
|
266
|
-
return {committedReplacementId: committedReplacementId ?? null, ownerClaimed: Boolean(ownerClient), retirementPending: Boolean(retiringClient)}
|
|
394
|
+
return {committedReplacementId: committedReplacementId ?? null, ownerClaimed: Boolean(ownerClient), retirementFailed, retirementPending: Boolean(retiringClient), retirementReady: retiringListenerReady}
|
|
267
395
|
}
|
|
268
396
|
|
|
269
397
|
if (request.command === "begin-owner-mutation") {
|
|
@@ -284,6 +412,8 @@ async function execute(request, socket) {
|
|
|
284
412
|
}
|
|
285
413
|
|
|
286
414
|
if (request.command === "prepare-owner-replacement") {
|
|
415
|
+
if (retirementFailed) throw new Error("Guardian owner replacement is fenced after incomplete retired-listener state transfer")
|
|
416
|
+
if (retiringClient) throw new Error("Guardian owner replacement cannot prepare while listener retirement is pending")
|
|
287
417
|
if (socket === ownerClient) throw new Error("Committed owner cannot prepare itself as a replacement")
|
|
288
418
|
if (replacementClient && replacementClient !== socket) throw new Error("Another owner replacement candidate is already prepared")
|
|
289
419
|
if (ownerMutationClient) throw new Error("Owner replacement cannot prepare while an owner mutation is in progress")
|
|
@@ -295,8 +425,18 @@ async function execute(request, socket) {
|
|
|
295
425
|
throw new Error("Owner replacement authority fence does not match the guardian's committed authority")
|
|
296
426
|
}
|
|
297
427
|
if (request.nextAuthority === undefined) throw new Error("Owner replacement requires the requested authority")
|
|
428
|
+
if (!ownerClient) {
|
|
429
|
+
cancelOwnerRestart()
|
|
430
|
+
const supersededRestartChild = ownerRestartChild
|
|
431
|
+
|
|
432
|
+
clearOwnerRestartTracking()
|
|
433
|
+
if (supersededRestartChild) killDetachedProcessGroup(supersededRestartChild)
|
|
434
|
+
}
|
|
298
435
|
if (!replacementId) replacementId = crypto.randomUUID()
|
|
436
|
+
retirementFailed = false
|
|
437
|
+
retiringListenerReady = false
|
|
299
438
|
replacementClient = socket
|
|
439
|
+
replacementOwnerPid = request.ownerPid
|
|
300
440
|
replacementAuthority = request.nextAuthority
|
|
301
441
|
replacementRevision = ownerRevision
|
|
302
442
|
if (ownerClient && !ownerClient.destroyed) ownerClient.write(`${JSON.stringify({event: "replacement-prepared", replacementId})}\n`)
|
|
@@ -309,7 +449,7 @@ async function execute(request, socket) {
|
|
|
309
449
|
if (request.ownerState === undefined || !isDeepStrictEqual(ownerAuthority(request.ownerState), replacementAuthority)) {
|
|
310
450
|
throw new Error("Prepared replacement state does not match the requested authority")
|
|
311
451
|
}
|
|
312
|
-
replacementOwnerState = request.ownerState
|
|
452
|
+
replacementOwnerState = mergeOwnerConnectionState(ownerState, request.ownerState)
|
|
313
453
|
if (!ownerClient) commitReplacement()
|
|
314
454
|
return {committed: ownerClient === socket}
|
|
315
455
|
}
|
|
@@ -320,30 +460,94 @@ async function execute(request, socket) {
|
|
|
320
460
|
return {aborted: true}
|
|
321
461
|
}
|
|
322
462
|
|
|
463
|
+
if (request.command === "prepare-retired-owner-listener-handoff") {
|
|
464
|
+
requireReplacement(socket, request)
|
|
465
|
+
requireRetiredOwnerReplacement(request)
|
|
466
|
+
await requireMissingOwnerControlPath(ownerState)
|
|
467
|
+
requireReplacement(socket, request)
|
|
468
|
+
if (!ownerClient || ownerClient.destroyed) throw new Error("Retired owner listener handoff requires the connected committed owner")
|
|
469
|
+
if (retiringClient) throw new Error("Another owner listener retirement is already pending")
|
|
470
|
+
retiringClient = ownerClient
|
|
471
|
+
retiringReplacementId = /** @type {string} */ (request.replacementId)
|
|
472
|
+
retiredListenerClients.set(ownerClient, {
|
|
473
|
+
listenerStateComplete: false,
|
|
474
|
+
localSources: new Map(),
|
|
475
|
+
replacementId: /** @type {string} */ (request.replacementId),
|
|
476
|
+
successor: socket
|
|
477
|
+
})
|
|
478
|
+
ownerClient.write(`${JSON.stringify({event: "replacement-listener-handoff-requested", replacementId: request.replacementId})}\n`)
|
|
479
|
+
return {prepared: true}
|
|
480
|
+
}
|
|
481
|
+
|
|
323
482
|
if (request.command === "commit-retired-owner-replacement") {
|
|
324
483
|
requireReplacement(socket, request)
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
}
|
|
329
|
-
if (!replacementOwnerState) throw new Error("Retired owner replacement transaction is not staged")
|
|
330
|
-
if (!isDeepStrictEqual(ownerAuthority(ownerState), replacementAuthority)) throw new Error("Retired owner replacement requires unchanged owner authority")
|
|
331
|
-
const controlPath = ownerControlPath(ownerState)
|
|
484
|
+
requireRetiredOwnerReplacement(request)
|
|
485
|
+
await requireMissingOwnerControlPath(ownerState)
|
|
486
|
+
const listener = retiringClient
|
|
332
487
|
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
throw new Error(`Retired owner control socket ${controlPath} still exists`)
|
|
336
|
-
} catch (error) {
|
|
337
|
-
if (!error || typeof error !== "object" || !("code" in error) || error.code !== "ENOENT") throw error
|
|
488
|
+
if (!listener || listener !== ownerClient || listener.destroyed || !retiringListenerReady || retiredListenerClients.get(listener)?.successor !== socket) {
|
|
489
|
+
throw new Error("Retired owner replacement requires a complete prepared listener handoff")
|
|
338
490
|
}
|
|
339
491
|
commitReplacement()
|
|
340
|
-
|
|
492
|
+
if (ownerClient) {
|
|
493
|
+
const retirement = retiredListenerClients.get(listener)
|
|
494
|
+
|
|
495
|
+
if (!retirement) throw new Error("Retired owner replacement lost its prepared listener handoff")
|
|
496
|
+
retirement.successor = ownerClient
|
|
497
|
+
listener.write(`${JSON.stringify({event: "replacement-retirement-requested", replacementId: request.replacementId})}\n`)
|
|
498
|
+
}
|
|
341
499
|
return {committed: true}
|
|
342
500
|
}
|
|
343
501
|
|
|
502
|
+
if (request.command === "publish-owner-connection-state") {
|
|
503
|
+
const retirement = retiredListenerClients.get(socket)
|
|
504
|
+
|
|
505
|
+
if (!retirement || retirement.replacementId !== request.replacementId) throw new Error("Owner connection state requires the exact retired listener transaction")
|
|
506
|
+
if (typeof request.sourceId !== "string" || !request.sourceId || typeof request.releaseId !== "string" || !request.connections || typeof request.connections !== "object" || Array.isArray(request.connections)) {
|
|
507
|
+
throw new Error("Owner connection state requires a source, release, and connection counts")
|
|
508
|
+
}
|
|
509
|
+
const connections = request.connections
|
|
510
|
+
const http = connections.http
|
|
511
|
+
const websocket = connections.websocket
|
|
512
|
+
|
|
513
|
+
if (typeof http !== "number" || !Number.isSafeInteger(http) || http < 0 || typeof websocket !== "number" || !Number.isSafeInteger(websocket) || websocket < 0) {
|
|
514
|
+
throw new Error("Owner connection state requires non-negative integer counts")
|
|
515
|
+
}
|
|
516
|
+
const normalized = {http, websocket}
|
|
517
|
+
|
|
518
|
+
if (request.localSource) setRetiredLocalSource(retirement, request.sourceId, request.releaseId, normalized)
|
|
519
|
+
applyOwnerConnectionState(ownerState, request.sourceId, request.releaseId, normalized)
|
|
520
|
+
if (replacementOwnerState) applyOwnerConnectionState(replacementOwnerState, request.sourceId, request.releaseId, normalized)
|
|
521
|
+
publishOwnerConnectionState(retirement.successor, request.sourceId, request.releaseId, normalized)
|
|
522
|
+
return {published: true}
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
if (request.command === "complete-owner-listener-retirement") {
|
|
526
|
+
if (retiredListenerClients.get(socket)?.replacementId !== request.replacementId || retiringClient !== socket) {
|
|
527
|
+
throw new Error("Owner listener retirement completion requires the exact retired listener transaction")
|
|
528
|
+
}
|
|
529
|
+
retiringListenerReady = true
|
|
530
|
+
const retirement = retiredListenerClients.get(socket)
|
|
531
|
+
|
|
532
|
+
if (!retirement) throw new Error("Owner listener retirement completion lost its transaction")
|
|
533
|
+
retirement.listenerStateComplete = true
|
|
534
|
+
const successor = retirement.successor
|
|
535
|
+
|
|
536
|
+
if (successor && !successor.destroyed) {
|
|
537
|
+
successor.write(`${JSON.stringify({event: "replacement-listeners-retired", replacementId: request.replacementId})}\n`)
|
|
538
|
+
}
|
|
539
|
+
return {completed: true}
|
|
540
|
+
}
|
|
541
|
+
|
|
344
542
|
if (request.command === "commit-owner-replacement") {
|
|
345
543
|
requireOwner(socket, request.command)
|
|
346
544
|
if (!replacementClient || request.replacementId !== replacementId || !replacementOwnerState) throw new Error("Owner replacement transaction is not the prepared ready candidate")
|
|
545
|
+
retiredListenerClients.set(socket, {
|
|
546
|
+
listenerStateComplete: false,
|
|
547
|
+
localSources: new Map(),
|
|
548
|
+
replacementId: /** @type {string} */ (request.replacementId),
|
|
549
|
+
successor: replacementClient
|
|
550
|
+
})
|
|
347
551
|
commitReplacement()
|
|
348
552
|
return {committed: true}
|
|
349
553
|
}
|
|
@@ -355,7 +559,15 @@ async function execute(request, socket) {
|
|
|
355
559
|
}
|
|
356
560
|
|
|
357
561
|
if (request.command === "finalize-owner-replacement") {
|
|
358
|
-
|
|
562
|
+
const incumbentRetirement = retiredListenerClients.get(socket)
|
|
563
|
+
const incumbentFinalization = retiringClient === socket && (!incumbentRetirement || incumbentRetirement.listenerStateComplete)
|
|
564
|
+
const candidateFinalization = ownerClient === socket && retiringListenerReady && !retirementFailed
|
|
565
|
+
const alreadyFinalized = ownerClient === socket && !retiringClient && request.replacementId === committedReplacementId && !retirementFailed
|
|
566
|
+
|
|
567
|
+
if (alreadyFinalized) return {finalized: true}
|
|
568
|
+
if ((!incumbentFinalization && !candidateFinalization) || request.replacementId !== retiringReplacementId) {
|
|
569
|
+
throw new Error("Owner replacement retirement finalization requires the committed listener transaction")
|
|
570
|
+
}
|
|
359
571
|
finalizeReplacementRetirement()
|
|
360
572
|
return {finalized: true}
|
|
361
573
|
}
|
|
@@ -424,7 +636,7 @@ async function execute(request, socket) {
|
|
|
424
636
|
: new ManagedProcess(managedDefinition)
|
|
425
637
|
|
|
426
638
|
managedProcess.on("log", (entry) => {
|
|
427
|
-
broadcast({entry, event: "process-log", key: request.key
|
|
639
|
+
broadcast({entry, event: "process-log", key: request.key})
|
|
428
640
|
})
|
|
429
641
|
if (recoversLegacyProcess && "recover" in managedProcess && typeof managedProcess.recover === "function") await managedProcess.recover()
|
|
430
642
|
|
|
@@ -443,6 +655,9 @@ async function execute(request, socket) {
|
|
|
443
655
|
await record.process.start(request.reason, request.lifecycleRole)
|
|
444
656
|
} else if (request.command === "activate") {
|
|
445
657
|
await record.process.activateStrict()
|
|
658
|
+
} else if (request.command === "reactivate" || request.command === "reactivate-with-command") {
|
|
659
|
+
await record.process.reactivateStrict()
|
|
660
|
+
record.desired = true
|
|
446
661
|
} else if (request.command === "quiesce") {
|
|
447
662
|
record.desired = false
|
|
448
663
|
await record.process.quiesceStrict()
|
|
@@ -458,7 +673,7 @@ async function execute(request, socket) {
|
|
|
458
673
|
} else if (request.command === "update") {
|
|
459
674
|
if (!request.definition || !request.provenance) throw new Error("Guardian update requires definition and provenance")
|
|
460
675
|
if (record.provenance !== request.previousProvenance) throw new Error(`Guardian provenance mismatch for ${request.key}`)
|
|
461
|
-
record.process.updateDefinition({
|
|
676
|
+
await record.process.updateDefinition({
|
|
462
677
|
...request.definition,
|
|
463
678
|
lifecycle: request.definition.lifecycle || {drainTimeoutMs: 0},
|
|
464
679
|
logger: record.process.logger,
|
|
@@ -468,6 +683,10 @@ async function execute(request, socket) {
|
|
|
468
683
|
stopSignal: request.definition.stopSignal || "SIGTERM"
|
|
469
684
|
})
|
|
470
685
|
record.provenance = request.provenance
|
|
686
|
+
if (request.ownerState !== undefined) {
|
|
687
|
+
ownerState = mergeOwnerConnectionState(ownerState, request.ownerState)
|
|
688
|
+
committedReplacementId = undefined
|
|
689
|
+
}
|
|
471
690
|
} else if (request.command === "status") {
|
|
472
691
|
return record.process.status()
|
|
473
692
|
} else {
|
|
@@ -490,17 +709,83 @@ function requireOwner(socket, command) {
|
|
|
490
709
|
if (ownerClient !== socket) throw new Error(`Guardian ${command} requires the committed owner`)
|
|
491
710
|
}
|
|
492
711
|
|
|
493
|
-
/**
|
|
494
|
-
|
|
495
|
-
|
|
712
|
+
/**
|
|
713
|
+
* Releases authority only after every accepted request from the disconnected owner settles.
|
|
714
|
+
* @param {net.Socket} socket - Disconnected owner socket.
|
|
715
|
+
*/
|
|
716
|
+
function releaseClosedOwner(socket) {
|
|
717
|
+
if (ownerClient !== socket) return
|
|
718
|
+
const incompleteRestartChild = ownerRestartChild?.pid === ownerClientPid ? ownerRestartChild : undefined
|
|
719
|
+
|
|
720
|
+
clearOwnerLocalListenerSource()
|
|
721
|
+
rememberDisconnectedOwnerSuccessors(socket)
|
|
722
|
+
ownerClient = undefined
|
|
723
|
+
ownerClientPid = undefined
|
|
724
|
+
if (ownerMutationClient === socket) {
|
|
725
|
+
ownerMutationClient = undefined
|
|
726
|
+
ownerMutationId = undefined
|
|
727
|
+
}
|
|
728
|
+
if (replacementClient && replacementOwnerState) commitReplacement()
|
|
729
|
+
else {
|
|
730
|
+
if (incompleteRestartChild) {
|
|
731
|
+
reportOwnerRestartFailure(ownerRecoveryDefinition(), {code: "OWNER_DISCONNECTED"})
|
|
732
|
+
killDetachedProcessGroup(incompleteRestartChild)
|
|
733
|
+
clearOwnerRestartTracking(incompleteRestartChild)
|
|
734
|
+
ownerRestartRetryDelayMs = OWNER_RESTART_RETRY_MS
|
|
735
|
+
}
|
|
736
|
+
if (replacementClient) abortReplacement("Committed owner disconnected before replacement candidate was ready", false)
|
|
737
|
+
if (retirementFailed) return
|
|
738
|
+
if (!shuttingDown) {
|
|
739
|
+
grantNextOwner()
|
|
740
|
+
const retryDelayMs = ownerRestartRetryDelayMs
|
|
741
|
+
|
|
742
|
+
ownerRestartRetryDelayMs = undefined
|
|
743
|
+
scheduleOwnerRestart(retryDelayMs)
|
|
744
|
+
}
|
|
745
|
+
}
|
|
746
|
+
}
|
|
747
|
+
|
|
748
|
+
/**
|
|
749
|
+
* Completes one accepted owner update and applies a deferred disconnect boundary.
|
|
750
|
+
* @param {net.Socket} socket - Requesting owner socket.
|
|
751
|
+
*/
|
|
752
|
+
function completeOwnerUpdate(socket) {
|
|
753
|
+
const remaining = (ownerUpdatesInFlight.get(socket) ?? 1) - 1
|
|
754
|
+
|
|
755
|
+
if (remaining > 0) {
|
|
756
|
+
ownerUpdatesInFlight.set(socket, remaining)
|
|
757
|
+
return
|
|
758
|
+
}
|
|
759
|
+
ownerUpdatesInFlight.delete(socket)
|
|
760
|
+
if (socket.destroyed) releaseClosedOwner(socket)
|
|
761
|
+
}
|
|
496
762
|
|
|
497
|
-
|
|
498
|
-
|
|
763
|
+
/**
|
|
764
|
+
* @param {string} reason - Abort diagnostic.
|
|
765
|
+
* @param {boolean} [scheduleRecovery] - Whether this call owns recovery scheduling.
|
|
766
|
+
*/
|
|
767
|
+
function abortReplacement(reason, scheduleRecovery = true) {
|
|
768
|
+
const abortedReplacementId = replacementId
|
|
769
|
+
|
|
770
|
+
if (replacementClient && !replacementClient.destroyed) replacementClient.write(`${JSON.stringify({event: "replacement-aborted", reason})}\n`)
|
|
771
|
+
if (ownerClient && !ownerClient.destroyed) ownerClient.write(`${JSON.stringify({event: "replacement-aborted", reason})}\n`)
|
|
772
|
+
if (retiringClient && retiringClient === ownerClient && retiringReplacementId === abortedReplacementId) {
|
|
773
|
+
const retirement = retiredListenerClients.get(retiringClient)
|
|
774
|
+
|
|
775
|
+
if (retirement) disconnectedOwnerSuccessors.delete(retirement)
|
|
776
|
+
retiredListenerClients.delete(retiringClient)
|
|
777
|
+
retiringClient = undefined
|
|
778
|
+
retiringReplacementId = undefined
|
|
779
|
+
retiringListenerReady = false
|
|
780
|
+
retirementFailed = false
|
|
781
|
+
}
|
|
499
782
|
replacementClient = undefined
|
|
500
783
|
replacementId = undefined
|
|
501
784
|
replacementAuthority = undefined
|
|
502
785
|
replacementOwnerState = undefined
|
|
786
|
+
replacementOwnerPid = undefined
|
|
503
787
|
replacementRevision = undefined
|
|
788
|
+
if (scheduleRecovery && !ownerClient) scheduleOwnerRestart()
|
|
504
789
|
}
|
|
505
790
|
|
|
506
791
|
/** Atomically promotes the prepared client and its complete transferable state. */
|
|
@@ -510,14 +795,31 @@ function commitReplacement() {
|
|
|
510
795
|
const committedClient = replacementClient
|
|
511
796
|
const committedId = replacementId
|
|
512
797
|
|
|
798
|
+
cancelOwnerRestart()
|
|
799
|
+
const supersededRestartChild = ownerRestartChild
|
|
800
|
+
const restartChildIsIncumbent = Boolean(previousOwner && supersededRestartChild?.pid === ownerClientPid)
|
|
801
|
+
|
|
802
|
+
clearOwnerRestartTracking()
|
|
803
|
+
if (supersededRestartChild && !restartChildIsIncumbent) killDetachedProcessGroup(supersededRestartChild)
|
|
513
804
|
ownerClient = committedClient
|
|
805
|
+
ownerClientPid = replacementOwnerPid
|
|
806
|
+
ownerRestartRetryDelayMs = undefined
|
|
514
807
|
ownerState = replacementOwnerState
|
|
515
808
|
ownerRevision += 1
|
|
516
809
|
committedReplacementId = committedId
|
|
810
|
+
for (const waiter of claimWaiters.splice(0)) {
|
|
811
|
+
if (isDeepStrictEqual(waiter.authority, ownerAuthority(ownerState))) {
|
|
812
|
+
claimWaiters.push(waiter)
|
|
813
|
+
} else {
|
|
814
|
+
clearTimeout(waiter.timer)
|
|
815
|
+
waiter.reject(new Error("Durable owner authority changed while the claim was queued"))
|
|
816
|
+
}
|
|
817
|
+
}
|
|
517
818
|
replacementClient = undefined
|
|
518
819
|
replacementId = undefined
|
|
519
820
|
replacementAuthority = undefined
|
|
520
821
|
replacementOwnerState = undefined
|
|
822
|
+
replacementOwnerPid = undefined
|
|
521
823
|
replacementRevision = undefined
|
|
522
824
|
if (previousOwner && !previousOwner.destroyed) {
|
|
523
825
|
retiringClient = previousOwner
|
|
@@ -535,6 +837,8 @@ function finalizeReplacementRetirement() {
|
|
|
535
837
|
|
|
536
838
|
retiringClient = undefined
|
|
537
839
|
retiringReplacementId = undefined
|
|
840
|
+
retiringListenerReady = false
|
|
841
|
+
retirementFailed = false
|
|
538
842
|
if (!committedClient || !committedId) return
|
|
539
843
|
publishReplacementCommitted(committedClient, committedId)
|
|
540
844
|
if (previousOwner && !previousOwner.destroyed) previousOwner.write(`${JSON.stringify({event: "replacement-retired", replacementId: committedId})}\n`)
|
|
@@ -568,6 +872,144 @@ function requireProcess(request) {
|
|
|
568
872
|
return record
|
|
569
873
|
}
|
|
570
874
|
|
|
875
|
+
/**
|
|
876
|
+
* @param {RetiredListenerTransaction} retirement - Retired listener transaction.
|
|
877
|
+
* @param {string} sourceId - Physically-owned source.
|
|
878
|
+
* @param {string} releaseId - Retained release.
|
|
879
|
+
* @param {{http: number, websocket: number}} connections - Exact live counts.
|
|
880
|
+
*/
|
|
881
|
+
function setRetiredLocalSource(retirement, sourceId, releaseId, connections) {
|
|
882
|
+
const releases = retirement.localSources.get(sourceId) || new Map()
|
|
883
|
+
|
|
884
|
+
if (connections.http + connections.websocket === 0) releases.delete(releaseId)
|
|
885
|
+
else releases.set(releaseId, connections)
|
|
886
|
+
if (releases.size === 0) retirement.localSources.delete(sourceId)
|
|
887
|
+
else retirement.localSources.set(sourceId, releases)
|
|
888
|
+
}
|
|
889
|
+
|
|
890
|
+
/**
|
|
891
|
+
* @param {net.Socket | undefined} firstSuccessor - First daemon that inherited the source.
|
|
892
|
+
* @param {string} sourceId - Stable listener source.
|
|
893
|
+
* @param {string} releaseId - Retained release.
|
|
894
|
+
* @param {{http: number, websocket: number}} connections - Exact live counts.
|
|
895
|
+
* @param {Set<net.Socket>} [delivered] - Successors that already received this update.
|
|
896
|
+
*/
|
|
897
|
+
function publishOwnerConnectionState(firstSuccessor, sourceId, releaseId, connections, delivered = new Set()) {
|
|
898
|
+
let successor = firstSuccessor
|
|
899
|
+
|
|
900
|
+
while (successor && !delivered.has(successor)) {
|
|
901
|
+
delivered.add(successor)
|
|
902
|
+
if (!successor.destroyed) {
|
|
903
|
+
successor.write(`${JSON.stringify({connections, event: "owner-connection-state", releaseId, sourceId})}\n`)
|
|
904
|
+
}
|
|
905
|
+
successor = retiredListenerClients.get(successor)?.successor
|
|
906
|
+
}
|
|
907
|
+
}
|
|
908
|
+
|
|
909
|
+
/** @param {RetiredListenerTransaction} retirement - Disconnected completed listener. */
|
|
910
|
+
function publishClosedLocalSources(retirement) {
|
|
911
|
+
for (const [sourceId, releases] of retirement.localSources) {
|
|
912
|
+
for (const releaseId of releases.keys()) {
|
|
913
|
+
const connections = {http: 0, websocket: 0}
|
|
914
|
+
const delivered = /** @type {Set<net.Socket>} */ (new Set())
|
|
915
|
+
|
|
916
|
+
applyOwnerConnectionState(ownerState, sourceId, releaseId, connections)
|
|
917
|
+
if (replacementOwnerState) {
|
|
918
|
+
applyOwnerConnectionState(replacementOwnerState, sourceId, releaseId, connections)
|
|
919
|
+
publishOwnerConnectionState(replacementClient, sourceId, releaseId, connections, delivered)
|
|
920
|
+
}
|
|
921
|
+
publishOwnerConnectionState(retirement.successor, sourceId, releaseId, connections, delivered)
|
|
922
|
+
}
|
|
923
|
+
}
|
|
924
|
+
}
|
|
925
|
+
|
|
926
|
+
/**
|
|
927
|
+
* @param {import("./json.js").JsonValue | undefined} state - Candidate or committed private owner state.
|
|
928
|
+
* @param {string} sourceId - Stable listener source.
|
|
929
|
+
* @param {string} releaseId - Retained release.
|
|
930
|
+
* @param {{http: number, websocket: number}} connections - Exact live counts.
|
|
931
|
+
*/
|
|
932
|
+
function applyOwnerConnectionState(state, sourceId, releaseId, connections) {
|
|
933
|
+
if (!state || typeof state !== "object" || Array.isArray(state)) throw new Error("Guardian owner connection state requires transferable owner state")
|
|
934
|
+
const transferable = /** @type {{listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>}} */ (state)
|
|
935
|
+
const sources = transferable.listenerConnectionSources || {}
|
|
936
|
+
const releases = sources[sourceId] || {}
|
|
937
|
+
|
|
938
|
+
if (connections.http + connections.websocket === 0) {
|
|
939
|
+
delete releases[releaseId]
|
|
940
|
+
} else {
|
|
941
|
+
releases[releaseId] = connections
|
|
942
|
+
}
|
|
943
|
+
if (Object.keys(releases).length === 0) delete sources[sourceId]
|
|
944
|
+
else sources[sourceId] = releases
|
|
945
|
+
transferable.listenerConnectionSources = sources
|
|
946
|
+
}
|
|
947
|
+
|
|
948
|
+
/** Clears the physically-owned source before another daemon reconstructs committed state. */
|
|
949
|
+
function clearOwnerLocalListenerSource() {
|
|
950
|
+
if (!ownerState || typeof ownerState !== "object" || Array.isArray(ownerState)) return
|
|
951
|
+
const transferable = /** @type {{listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>, listenerSourceId?: string}} */ (ownerState)
|
|
952
|
+
const sourceId = transferable.listenerSourceId
|
|
953
|
+
|
|
954
|
+
if (!sourceId) return
|
|
955
|
+
for (const releaseId of Object.keys(transferable.listenerConnectionSources?.[sourceId] || {})) {
|
|
956
|
+
const connections = {http: 0, websocket: 0}
|
|
957
|
+
|
|
958
|
+
applyOwnerConnectionState(ownerState, sourceId, releaseId, connections)
|
|
959
|
+
if (replacementOwnerState) {
|
|
960
|
+
applyOwnerConnectionState(replacementOwnerState, sourceId, releaseId, connections)
|
|
961
|
+
publishOwnerConnectionState(replacementClient, sourceId, releaseId, connections)
|
|
962
|
+
}
|
|
963
|
+
}
|
|
964
|
+
}
|
|
965
|
+
|
|
966
|
+
/**
|
|
967
|
+
* Preserves guardian-confirmed source updates while accepting the publisher's own local source.
|
|
968
|
+
* @param {import("./json.js").JsonValue | undefined} previousState - Current guardian state.
|
|
969
|
+
* @param {import("./json.js").JsonValue} nextState - Incoming owner publication.
|
|
970
|
+
* @returns {import("./json.js").JsonValue} Publication with guardian source state merged in.
|
|
971
|
+
*/
|
|
972
|
+
function mergeOwnerConnectionState(previousState, nextState) {
|
|
973
|
+
if (!nextState || typeof nextState !== "object" || Array.isArray(nextState)) throw new Error("Guardian owner publication requires transferable owner state")
|
|
974
|
+
const next = /** @type {{listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>, listenerSourceId?: string}} */ (nextState)
|
|
975
|
+
const publishesSources = next.listenerConnectionSources !== undefined
|
|
976
|
+
const localSourceId = next.listenerSourceId
|
|
977
|
+
|
|
978
|
+
if (!previousState || typeof previousState !== "object" || Array.isArray(previousState) || !localSourceId) return nextState
|
|
979
|
+
const previous = /** @type {{listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>}} */ (previousState)
|
|
980
|
+
const sources = /** @type {Record<string, Record<string, {http: number, websocket: number}>>} */ ({})
|
|
981
|
+
const localReleases = next.listenerConnectionSources?.[localSourceId]
|
|
982
|
+
|
|
983
|
+
if (localReleases && Object.keys(localReleases).length > 0) sources[localSourceId] = localReleases
|
|
984
|
+
for (const [sourceId, releases] of Object.entries(previous.listenerConnectionSources || {})) {
|
|
985
|
+
if (sourceId !== localSourceId) sources[sourceId] = releases
|
|
986
|
+
}
|
|
987
|
+
if (publishesSources || Object.keys(sources).length > 0) next.listenerConnectionSources = sources
|
|
988
|
+
return nextState
|
|
989
|
+
}
|
|
990
|
+
|
|
991
|
+
/** @param {GuardianRequest} request - Staged same-authority listener handoff request. */
|
|
992
|
+
function requireRetiredOwnerReplacement(request) {
|
|
993
|
+
requireProcess(request)
|
|
994
|
+
if (!committedOwnerProcessKeys(ownerState).has(/** @type {string} */ (request.key))) {
|
|
995
|
+
throw new Error(`Guardian process ${request.key} does not belong to the committed owner`)
|
|
996
|
+
}
|
|
997
|
+
if (!replacementOwnerState) throw new Error("Retired owner replacement transaction is not staged")
|
|
998
|
+
if (!isDeepStrictEqual(ownerAuthority(ownerState), replacementAuthority)) throw new Error("Retired owner replacement requires unchanged owner authority")
|
|
999
|
+
}
|
|
1000
|
+
|
|
1001
|
+
/** @param {import("./json.js").JsonValue | undefined} state - Committed transferable state. */
|
|
1002
|
+
async function requireMissingOwnerControlPath(state) {
|
|
1003
|
+
const controlPath = ownerControlPath(state)
|
|
1004
|
+
|
|
1005
|
+
try {
|
|
1006
|
+
await fs.lstat(controlPath)
|
|
1007
|
+
throw new Error(`Retired owner control socket ${controlPath} still exists`)
|
|
1008
|
+
} catch (error) {
|
|
1009
|
+
if (!error || typeof error !== "object" || !("code" in error) || error.code !== "ENOENT") throw error
|
|
1010
|
+
}
|
|
1011
|
+
}
|
|
1012
|
+
|
|
571
1013
|
/**
|
|
572
1014
|
* @param {import("./json.js").JsonValue} state - Transfer state.
|
|
573
1015
|
* @returns {import("./json.js").JsonValue} Embedded authority fence.
|
|
@@ -592,6 +1034,31 @@ function ownerControlPath(state) {
|
|
|
592
1034
|
return control.path
|
|
593
1035
|
}
|
|
594
1036
|
|
|
1037
|
+
/**
|
|
1038
|
+
* @param {import("./json.js").JsonValue | undefined} state - Committed transferable state.
|
|
1039
|
+
* @returns {string} Exact configured public state path.
|
|
1040
|
+
*/
|
|
1041
|
+
function ownerStatePath(state) {
|
|
1042
|
+
if (!state || typeof state !== "object" || Array.isArray(state) || !("config" in state)) throw new Error("Guardian owner state is missing its committed config")
|
|
1043
|
+
const config = state.config
|
|
1044
|
+
|
|
1045
|
+
if (!config || typeof config !== "object" || Array.isArray(config) || !("statePath" in config) || typeof config.statePath !== "string") throw new Error("Guardian owner config is missing its state path")
|
|
1046
|
+
return config.statePath
|
|
1047
|
+
}
|
|
1048
|
+
|
|
1049
|
+
/** @param {import("./json.js").JsonValue | undefined} snapshot - Candidate public recovery snapshot. */
|
|
1050
|
+
function assertBridgeRecoverySnapshot(snapshot) {
|
|
1051
|
+
if (!snapshot || typeof snapshot !== "object" || Array.isArray(snapshot) || !("recovery" in snapshot)) throw new Error("Legacy guardian owner claim requires a recovery snapshot")
|
|
1052
|
+
const recovery = snapshot.recovery
|
|
1053
|
+
|
|
1054
|
+
if (!recovery || typeof recovery !== "object" || Array.isArray(recovery) || !("guardian" in recovery)) throw new Error("Legacy guardian owner claim recovery snapshot is missing its guardian")
|
|
1055
|
+
const guardian = recovery.guardian
|
|
1056
|
+
|
|
1057
|
+
if (!guardian || typeof guardian !== "object" || Array.isArray(guardian) || guardian.socketPath !== socketPath || guardian.token !== token) {
|
|
1058
|
+
throw new Error("Legacy guardian owner claim recovery snapshot does not identify this guardian")
|
|
1059
|
+
}
|
|
1060
|
+
}
|
|
1061
|
+
|
|
595
1062
|
/**
|
|
596
1063
|
* Stops accepting connections and closes every authority channel except the response caller.
|
|
597
1064
|
* @param {net.Socket} caller - Shutdown requester retained until it receives the response.
|
|
@@ -624,16 +1091,75 @@ async function finishShutdown() {
|
|
|
624
1091
|
function broadcast(event) {
|
|
625
1092
|
const line = `${JSON.stringify(event)}\n`
|
|
626
1093
|
|
|
627
|
-
for (const client of clients)
|
|
1094
|
+
for (const client of clients) {
|
|
1095
|
+
if (client.destroyed) continue
|
|
1096
|
+
if (backpressuredClients.has(client)) {
|
|
1097
|
+
if (typeof event.key === "string") {
|
|
1098
|
+
const pending = pendingStatusEvents.get(client) || new Map()
|
|
1099
|
+
|
|
1100
|
+
if (event.event === "process-log") {
|
|
1101
|
+
const record = processes.get(event.key)
|
|
1102
|
+
|
|
1103
|
+
if (record) pending.set(event.key, `${JSON.stringify({event: "status", key: event.key, status: record.process.status()})}\n`)
|
|
1104
|
+
} else {
|
|
1105
|
+
pending.set(event.key, line)
|
|
1106
|
+
}
|
|
1107
|
+
pendingStatusEvents.set(client, pending)
|
|
1108
|
+
}
|
|
1109
|
+
continue
|
|
1110
|
+
}
|
|
1111
|
+
if (!client.write(line)) backpressuredClients.add(client)
|
|
1112
|
+
}
|
|
1113
|
+
}
|
|
1114
|
+
|
|
1115
|
+
/**
|
|
1116
|
+
* Flushes one latest status-bearing event per process after socket backpressure clears.
|
|
1117
|
+
* @param {net.Socket} client - Drained guardian client socket.
|
|
1118
|
+
*/
|
|
1119
|
+
function flushStatusEvents(client) {
|
|
1120
|
+
backpressuredClients.delete(client)
|
|
1121
|
+
const pending = pendingStatusEvents.get(client)
|
|
1122
|
+
|
|
1123
|
+
if (!pending || client.destroyed) return
|
|
1124
|
+
for (const [key, line] of pending) {
|
|
1125
|
+
pending.delete(key)
|
|
1126
|
+
if (!client.write(line)) {
|
|
1127
|
+
backpressuredClients.add(client)
|
|
1128
|
+
break
|
|
1129
|
+
}
|
|
1130
|
+
}
|
|
1131
|
+
if (pending.size === 0) pendingStatusEvents.delete(client)
|
|
1132
|
+
}
|
|
1133
|
+
|
|
1134
|
+
/** @param {net.Socket} owner - Disconnected or retiring committed owner. */
|
|
1135
|
+
function rememberDisconnectedOwnerSuccessors(owner) {
|
|
1136
|
+
for (const transaction of retiredListenerClients.values()) {
|
|
1137
|
+
if (transaction.successor === owner) disconnectedOwnerSuccessors.add(transaction)
|
|
1138
|
+
}
|
|
1139
|
+
}
|
|
1140
|
+
|
|
1141
|
+
/** @param {net.Socket} successor - Newly claimed recovery owner. */
|
|
1142
|
+
function reattachRetiredListenerSuccessors(successor) {
|
|
1143
|
+
for (const transaction of disconnectedOwnerSuccessors) transaction.successor = successor
|
|
1144
|
+
disconnectedOwnerSuccessors.clear()
|
|
628
1145
|
}
|
|
629
1146
|
|
|
630
1147
|
/** @returns {void} Grants the next queued owner claim. */
|
|
631
1148
|
function grantNextOwner() {
|
|
632
|
-
|
|
1149
|
+
let next = claimWaiters.shift()
|
|
1150
|
+
|
|
1151
|
+
while (next && ownerState !== undefined && !isDeepStrictEqual(next.authority, ownerAuthority(ownerState))) {
|
|
1152
|
+
clearTimeout(next.timer)
|
|
1153
|
+
next.reject(new Error("Owner recovery authority changed while the claim was queued"))
|
|
1154
|
+
next = claimWaiters.shift()
|
|
1155
|
+
}
|
|
633
1156
|
|
|
634
1157
|
if (!next) return
|
|
635
1158
|
clearTimeout(next.timer)
|
|
1159
|
+
acceptOwnerClaim(next.ownerPid)
|
|
636
1160
|
ownerClient = next.socket
|
|
1161
|
+
ownerClientPid = next.ownerPid
|
|
1162
|
+
reattachRetiredListenerSuccessors(next.socket)
|
|
637
1163
|
next.resolve({claimed: true})
|
|
638
1164
|
for (const waiter of claimWaiters.splice(0)) {
|
|
639
1165
|
clearTimeout(waiter.timer)
|
|
@@ -641,6 +1167,171 @@ function grantNextOwner() {
|
|
|
641
1167
|
}
|
|
642
1168
|
}
|
|
643
1169
|
|
|
1170
|
+
/** @param {number} [delayMs] - Explicit retry delay, otherwise the accepted reconnect grace. */
|
|
1171
|
+
function scheduleOwnerRestart(delayMs) {
|
|
1172
|
+
if (shuttingDown || ownerClient || replacementClient || ownerRestartChild || ownerRestartTimer) return
|
|
1173
|
+
const recovery = ownerRecoveryDefinition()
|
|
1174
|
+
|
|
1175
|
+
if (!recovery) return
|
|
1176
|
+
ownerRestartTimer = setTimeout(() => {
|
|
1177
|
+
ownerRestartTimer = undefined
|
|
1178
|
+
if (shuttingDown || ownerClient || replacementClient || ownerRestartChild) return
|
|
1179
|
+
const currentRecovery = ownerRecoveryDefinition()
|
|
1180
|
+
|
|
1181
|
+
if (!currentRecovery) return
|
|
1182
|
+
let stdoutFd
|
|
1183
|
+
let stderrFd
|
|
1184
|
+
let child
|
|
1185
|
+
|
|
1186
|
+
try {
|
|
1187
|
+
if (currentRecovery.command.logPath) {
|
|
1188
|
+
fsSync.mkdirSync(path.dirname(currentRecovery.command.logPath), {recursive: true})
|
|
1189
|
+
stdoutFd = fsSync.openSync(currentRecovery.command.logPath, "a")
|
|
1190
|
+
stderrFd = fsSync.openSync(currentRecovery.command.logPath, "a")
|
|
1191
|
+
}
|
|
1192
|
+
child = spawn(currentRecovery.command.executable, currentRecovery.command.args, {
|
|
1193
|
+
cwd: currentRecovery.command.cwd,
|
|
1194
|
+
detached: true,
|
|
1195
|
+
env: currentRecovery.command.env,
|
|
1196
|
+
stdio: currentRecovery.command.logPath ? ["ignore", stdoutFd, stderrFd] : ["ignore", "inherit", "inherit"]
|
|
1197
|
+
})
|
|
1198
|
+
} catch (error) {
|
|
1199
|
+
reportOwnerRestartFailure(currentRecovery, restartFailure(error instanceof Error ? error : String(error)))
|
|
1200
|
+
scheduleOwnerRestart(OWNER_RESTART_RETRY_MS)
|
|
1201
|
+
return
|
|
1202
|
+
} finally {
|
|
1203
|
+
if (stdoutFd !== undefined) fsSync.closeSync(stdoutFd)
|
|
1204
|
+
if (stderrFd !== undefined) fsSync.closeSync(stderrFd)
|
|
1205
|
+
}
|
|
1206
|
+
|
|
1207
|
+
ownerRestartChild = child
|
|
1208
|
+
let settled = false
|
|
1209
|
+
const retry = (/** @type {{code: string, exitCode?: number | null, signal?: ChildExitSignal}} */ failure) => {
|
|
1210
|
+
if (settled || ownerRestartChild !== child) return
|
|
1211
|
+
settled = true
|
|
1212
|
+
killDetachedProcessGroup(child)
|
|
1213
|
+
reportOwnerRestartFailure(currentRecovery, failure)
|
|
1214
|
+
clearOwnerRestartTracking(child)
|
|
1215
|
+
ownerRestartRetryDelayMs = OWNER_RESTART_RETRY_MS
|
|
1216
|
+
if (ownerClient && ownerClientPid === child.pid) ownerClient.destroy()
|
|
1217
|
+
else {
|
|
1218
|
+
ownerRestartRetryDelayMs = undefined
|
|
1219
|
+
scheduleOwnerRestart(OWNER_RESTART_RETRY_MS)
|
|
1220
|
+
}
|
|
1221
|
+
}
|
|
1222
|
+
|
|
1223
|
+
ownerRestartStartupTimer = setTimeout(() => {
|
|
1224
|
+
if (ownerRestartChild !== child) return
|
|
1225
|
+
retry({code: "STARTUP_TIMEOUT"})
|
|
1226
|
+
}, currentRecovery.startupTimeoutMs)
|
|
1227
|
+
ownerRestartStartupTimer.unref()
|
|
1228
|
+
child.once("error", (error) => retry(restartFailure(error)))
|
|
1229
|
+
child.once("exit", (code, signal) => retry({code: "EARLY_EXIT", exitCode: code, signal}))
|
|
1230
|
+
child.unref()
|
|
1231
|
+
}, delayMs ?? recovery.reconnectGraceMs)
|
|
1232
|
+
ownerRestartTimer.unref()
|
|
1233
|
+
}
|
|
1234
|
+
|
|
1235
|
+
/** Cancels recovery for a superseded owner command. */
|
|
1236
|
+
function cancelOwnerRestart() {
|
|
1237
|
+
if (!ownerRestartTimer) return
|
|
1238
|
+
clearTimeout(ownerRestartTimer)
|
|
1239
|
+
ownerRestartTimer = undefined
|
|
1240
|
+
}
|
|
1241
|
+
|
|
1242
|
+
/**
|
|
1243
|
+
* Accepts an exact claimant and retires any different in-flight recovery child.
|
|
1244
|
+
* @param {number | undefined} ownerPid - Exact claimant process PID.
|
|
1245
|
+
*/
|
|
1246
|
+
function acceptOwnerClaim(ownerPid) {
|
|
1247
|
+
cancelOwnerRestart()
|
|
1248
|
+
const child = ownerRestartChild
|
|
1249
|
+
|
|
1250
|
+
if (!child || child.pid === ownerPid) return
|
|
1251
|
+
clearOwnerRestartTracking(child)
|
|
1252
|
+
killDetachedProcessGroup(child)
|
|
1253
|
+
}
|
|
1254
|
+
|
|
1255
|
+
/**
|
|
1256
|
+
* Clears recovery-child startup tracking without disturbing a committed retired listener owner.
|
|
1257
|
+
* @param {import("node:child_process").ChildProcess} [expectedChild] - Optional exact child fence.
|
|
1258
|
+
* @returns {boolean} Whether matching tracking was cleared.
|
|
1259
|
+
*/
|
|
1260
|
+
function clearOwnerRestartTracking(expectedChild) {
|
|
1261
|
+
if (expectedChild && ownerRestartChild !== expectedChild) return false
|
|
1262
|
+
if (ownerRestartStartupTimer) clearTimeout(ownerRestartStartupTimer)
|
|
1263
|
+
ownerRestartStartupTimer = undefined
|
|
1264
|
+
ownerRestartChild = undefined
|
|
1265
|
+
return true
|
|
1266
|
+
}
|
|
1267
|
+
|
|
1268
|
+
/**
|
|
1269
|
+
* Records an owner restart failure without exposing the private command or environment.
|
|
1270
|
+
* @param {{command: {cwd?: string, logPath?: string, pidPath?: string}} | undefined} recovery - Accepted recovery definition.
|
|
1271
|
+
* @param {{code: string, exitCode?: number | null, signal?: ChildExitSignal}} failure - Secret-safe startup failure.
|
|
1272
|
+
*/
|
|
1273
|
+
function reportOwnerRestartFailure(recovery, failure) {
|
|
1274
|
+
const diagnostic = `${JSON.stringify({at: new Date().toISOString(), ...failure, message: "guardian failed to restart daemon"})}\n`
|
|
1275
|
+
|
|
1276
|
+
if (recovery?.command.logPath) {
|
|
1277
|
+
try {
|
|
1278
|
+
fsSync.appendFileSync(recovery.command.logPath, diagnostic)
|
|
1279
|
+
return
|
|
1280
|
+
} catch (logError) {
|
|
1281
|
+
process.stderr.write(`${JSON.stringify({at: new Date().toISOString(), code: errorCode(logError instanceof Error ? logError : String(logError)), message: "guardian failed to write daemon restart diagnostic"})}\n`)
|
|
1282
|
+
return
|
|
1283
|
+
}
|
|
1284
|
+
}
|
|
1285
|
+
process.stderr.write(diagnostic)
|
|
1286
|
+
}
|
|
1287
|
+
|
|
1288
|
+
/**
|
|
1289
|
+
* Terminates a detached recovery candidate and all descendants in its process group.
|
|
1290
|
+
* @param {import("node:child_process").ChildProcess} child - Exact detached process-group leader.
|
|
1291
|
+
*/
|
|
1292
|
+
function killDetachedProcessGroup(child) {
|
|
1293
|
+
if (!child.pid) return
|
|
1294
|
+
try {
|
|
1295
|
+
process.kill(-child.pid, "SIGKILL")
|
|
1296
|
+
} catch (error) {
|
|
1297
|
+
const failure = error instanceof Error ? error : String(error)
|
|
1298
|
+
|
|
1299
|
+
if (errorCode(failure) === "ESRCH") return
|
|
1300
|
+
child.kill("SIGKILL")
|
|
1301
|
+
process.stderr.write(`${JSON.stringify({at: new Date().toISOString(), code: errorCode(failure), message: "guardian failed to kill daemon restart process group"})}\n`)
|
|
1302
|
+
}
|
|
1303
|
+
}
|
|
1304
|
+
|
|
1305
|
+
/**
|
|
1306
|
+
* @param {Error | string} error - Spawn failure.
|
|
1307
|
+
* @returns {{code: string}} Secret-safe failure.
|
|
1308
|
+
*/
|
|
1309
|
+
function restartFailure(error) {
|
|
1310
|
+
return {code: errorCode(error)}
|
|
1311
|
+
}
|
|
1312
|
+
|
|
1313
|
+
/**
|
|
1314
|
+
* @param {Error | string} error - Error-like value.
|
|
1315
|
+
* @returns {string} Stable non-secret error code.
|
|
1316
|
+
*/
|
|
1317
|
+
function errorCode(error) {
|
|
1318
|
+
if (error && typeof error === "object" && "code" in error && (typeof error.code === "string" || typeof error.code === "number")) return String(error.code)
|
|
1319
|
+
return "UNKNOWN"
|
|
1320
|
+
}
|
|
1321
|
+
|
|
1322
|
+
/**
|
|
1323
|
+
* @returns {{command: {args: string[], cwd: string, env: Record<string, string>, executable: string, logPath?: string, pidPath?: string}, reconnectGraceMs: number, startupTimeoutMs: number} | undefined} Recovery definition.
|
|
1324
|
+
*/
|
|
1325
|
+
function ownerRecoveryDefinition() {
|
|
1326
|
+
if (!ownerState || typeof ownerState !== "object" || Array.isArray(ownerState) || !("recovery" in ownerState)) return
|
|
1327
|
+
const recovery = ownerState.recovery
|
|
1328
|
+
|
|
1329
|
+
if (!recovery || typeof recovery !== "object" || Array.isArray(recovery) || !("command" in recovery) || !("reconnectGraceMs" in recovery)) {
|
|
1330
|
+
throw new Error("Guardian owner recovery definition is incomplete")
|
|
1331
|
+
}
|
|
1332
|
+
return /** @type {{command: {args: string[], cwd: string, env: Record<string, string>, executable: string, logPath?: string, pidPath?: string}, reconnectGraceMs: number, startupTimeoutMs: number}} */ (recovery)
|
|
1333
|
+
}
|
|
1334
|
+
|
|
644
1335
|
/**
|
|
645
1336
|
* @param {Error | string} error - Error-like value.
|
|
646
1337
|
* @returns {string} Error message.
|