rollbridge 0.1.39 → 0.1.41

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,32 +1,48 @@
1
1
  // @ts-check
2
2
 
3
+ import fsSync from "node:fs"
3
4
  import fs from "node:fs/promises"
4
5
  import net from "node:net"
5
6
  import crypto from "node:crypto"
7
+ import {spawn} from "node:child_process"
6
8
  import {isDeepStrictEqual} from "node:util"
9
+ import path from "node:path"
7
10
  import GuardianClient from "./guardian-client.js"
8
11
  import ManagedProcess from "./managed-process.js"
12
+ import {writeState} from "./state-store.js"
13
+
14
+ const OWNER_RESTART_RETRY_MS = 1000
15
+
16
+ /** @typedef {import("node:child_process").ChildProcess["signalCode"]} ChildExitSignal */
9
17
 
10
18
  /**
11
19
  * @typedef {object} GuardianRequest
12
20
  * @property {string} command - Operation name.
21
+ * @property {{http?: number, websocket?: number}} [connections] - Retired listener counts.
13
22
  * @property {ConstructorParameters<typeof ManagedProcess>[0]} [definition] - Managed process definition.
14
23
  * @property {number} [graceMs] - Owner reconnection grace.
15
24
  * @property {number} id - Request id.
16
25
  * @property {string} [key] - Stable process key.
26
+ * @property {boolean} [localSource] - Whether the retired daemon physically owns the published source.
17
27
  * @property {{timeoutMs?: number}} [options] - Stop options.
18
28
  * @property {string} [previousProvenance] - Expected current provenance.
19
29
  * @property {string} [provenance] - New or registered provenance.
30
+ * @property {string} [releaseId] - Retained release identity.
31
+ * @property {string} [sourceId] - Stable retired-listener source identity.
20
32
  * @property {import("./json.js").JsonValue} [ownerState] - Private owner transfer state.
21
33
  * @property {string} [replacementId] - Prepared replacement transaction id.
22
34
  * @property {string} [mutationId] - Owner mutation lease id.
23
35
  * @property {string} [operation] - Owner mutation diagnostic name.
36
+ * @property {number} [ownerPid] - Exact claimant process PID.
37
+ * @property {import("./json.js").JsonValue} [recoverySnapshot] - Public bridge recovery snapshot.
38
+ * @property {string} [statePath] - Exact public recovery state path.
24
39
  * @property {import("./json.js").JsonValue} [authority] - Expected current authority.
25
40
  * @property {import("./json.js").JsonValue} [nextAuthority] - Requested replacement authority.
26
41
  * @property {import("./managed-process.js").ManagedProcessStartReason} [reason] - Start reason.
27
42
  * @property {import("./managed-process.js").LifecycleRole} [lifecycleRole] - Desired generation role restored during start.
28
43
  * @property {string} token - Authentication token.
29
44
  */
45
+ /** @typedef {{listenerStateComplete: boolean, localSources: Map<string, Map<string, {http: number, websocket: number}>>, replacementId: string, successor: net.Socket}} RetiredListenerTransaction */
30
46
 
31
47
  const [socketPath] = process.argv.slice(2)
32
48
 
@@ -51,10 +67,24 @@ if (legacyGuardian) await legacyGuardian.connect()
51
67
  const processes = new Map()
52
68
  /** @type {Set<net.Socket>} */
53
69
  const clients = new Set()
70
+ /** @type {Set<net.Socket>} */
71
+ const backpressuredClients = new Set()
72
+ /** @type {Map<net.Socket, Map<string, string>>} */
73
+ const pendingStatusEvents = new Map()
74
+ /** @type {Map<net.Socket, number>} */
75
+ const ownerUpdatesInFlight = new Map()
76
+ /** @type {Map<net.Socket, RetiredListenerTransaction>} */
77
+ const retiredListenerClients = new Map()
78
+ /** @type {Set<RetiredListenerTransaction>} */
79
+ const disconnectedOwnerSuccessors = new Set()
54
80
  /** @type {net.Socket | undefined} */
55
81
  let ownerClient
82
+ /** @type {number | undefined} */
83
+ let ownerClientPid
56
84
  /** @type {net.Socket | undefined} */
57
85
  let replacementClient
86
+ /** @type {number | undefined} */
87
+ let replacementOwnerPid
58
88
  /** @type {string | undefined} */
59
89
  let replacementId
60
90
  /** @type {string | undefined} */
@@ -77,15 +107,29 @@ let ownerMutationId
77
107
  let retiringClient
78
108
  /** @type {string | undefined} */
79
109
  let retiringReplacementId
110
+ let retiringListenerReady = false
111
+ let retirementFailed = false
80
112
  /** @type {Promise<Error | undefined> | undefined} */
81
113
  let legacyOwnerClaim
114
+ /** @type {import("./json.js").JsonValue | undefined} */
115
+ let legacyRecoverySnapshot
116
+ /** @type {string | undefined} */
117
+ let legacyRecoveryStatePath
82
118
  let shuttingDown = false
83
119
  /** @type {net.Socket | undefined} */
84
120
  let shutdownClient
85
121
  let shutdownFinalizing = false
86
122
  /** @type {Promise<void> | undefined} */
87
123
  let serverClosed
88
- /** @type {{reject: (error: Error) => void, resolve: (value: {claimed: boolean}) => void, socket: net.Socket, timer: ReturnType<typeof setTimeout>}[]} */
124
+ /** @type {import("node:child_process").ChildProcess | undefined} */
125
+ let ownerRestartChild
126
+ /** @type {ReturnType<typeof setTimeout> | undefined} */
127
+ let ownerRestartTimer
128
+ /** @type {ReturnType<typeof setTimeout> | undefined} */
129
+ let ownerRestartStartupTimer
130
+ /** @type {number | undefined} */
131
+ let ownerRestartRetryDelayMs
132
+ /** @type {{authority: import("./json.js").JsonValue, ownerPid: number | undefined, reject: (error: Error) => void, resolve: (value: {claimed: boolean}) => void, socket: net.Socket, timer: ReturnType<typeof setTimeout>}[]} */
89
133
  const claimWaiters = []
90
134
 
91
135
  const server = net.createServer((socket) => {
@@ -94,10 +138,15 @@ const server = net.createServer((socket) => {
94
138
  // An abruptly killed daemon can reset its private guardian connection. Keep the
95
139
  // durable guardian alive; the close handler below releases only that owner claim.
96
140
  socket.on("error", () => socket.destroy())
141
+ socket.on("drain", () => flushStatusEvents(socket))
97
142
  let buffer = ""
98
143
 
99
144
  socket.once("close", () => {
145
+ const controlLessRetirement = retiredListenerClients.get(socket)
146
+
100
147
  clients.delete(socket)
148
+ backpressuredClients.delete(socket)
149
+ pendingStatusEvents.delete(socket)
101
150
  const waiterIndex = claimWaiters.findIndex((waiter) => waiter.socket === socket)
102
151
 
103
152
  if (waiterIndex >= 0) {
@@ -105,21 +154,46 @@ const server = net.createServer((socket) => {
105
154
  clearTimeout(waiter.timer)
106
155
  waiter.reject(new Error("Owner claimant disconnected"))
107
156
  }
108
- if (ownerClient === socket) {
109
- ownerClient = undefined
110
- if (replacementClient && replacementOwnerState) commitReplacement()
111
- else {
112
- if (replacementClient) abortReplacement("Committed owner disconnected before replacement candidate was ready")
113
- if (!shuttingDown) grantNextOwner()
114
- }
115
- }
116
- if (ownerMutationClient === socket) {
157
+ if (ownerMutationClient === socket && ownerClient !== socket) {
117
158
  ownerMutationClient = undefined
118
159
  ownerMutationId = undefined
119
160
  }
120
- if (retiringClient === socket) finalizeReplacementRetirement()
161
+ if (controlLessRetirement) {
162
+ const preCommitHandoff = replacementClient && ownerClient === socket && retiringReplacementId === replacementId
163
+
164
+ if (preCommitHandoff) {
165
+ publishClosedLocalSources(controlLessRetirement)
166
+ if (!controlLessRetirement.successor.destroyed) {
167
+ controlLessRetirement.successor.write(`${JSON.stringify({event: "replacement-retirement-failed", reason: "Incumbent listener disconnected during the prepared handoff", replacementId: controlLessRetirement.replacementId})}\n`)
168
+ }
169
+ abortReplacement("Incumbent listener disconnected during the prepared handoff", false)
170
+ } else if (retiringClient === socket && !controlLessRetirement.listenerStateComplete) {
171
+ retirementFailed = true
172
+ retiringClient = undefined
173
+ retiringReplacementId = undefined
174
+ if (ownerClient && !ownerClient.destroyed) {
175
+ ownerClient.write(`${JSON.stringify({event: "replacement-retirement-failed", reason: "Retired listener disconnected before publishing complete connection state", replacementId: controlLessRetirement.replacementId})}\n`)
176
+ }
177
+ } else if (controlLessRetirement.listenerStateComplete) {
178
+ publishClosedLocalSources(controlLessRetirement)
179
+ if (retiringClient === socket) finalizeReplacementRetirement()
180
+ }
181
+ } else if (retiringClient === socket) finalizeReplacementRetirement()
182
+ if (controlLessRetirement && retiredListenerClients.has(socket)) {
183
+ for (const transaction of retiredListenerClients.values()) {
184
+ if (transaction.successor === socket) transaction.successor = controlLessRetirement.successor
185
+ }
186
+ disconnectedOwnerSuccessors.delete(controlLessRetirement)
187
+ retiredListenerClients.delete(socket)
188
+ }
189
+ if (ownerClient === socket && !ownerUpdatesInFlight.has(socket)) releaseClosedOwner(socket)
121
190
  if (replacementClient === socket) {
122
191
  abortReplacement("Replacement candidate disconnected before commit")
192
+ if (legacyGuardian && !legacyOwnerClaim && !ownerClient && !committedReplacementId) {
193
+ legacyGuardian.disconnect()
194
+ shuttingDown = true
195
+ beginSuccessfulShutdown(socket)
196
+ }
123
197
  }
124
198
  if (shutdownClient === socket) void finishShutdown()
125
199
  })
@@ -158,15 +232,22 @@ server.listen(socketPath, async () => {
158
232
  async function handleLine(socket, line) {
159
233
  /** @type {GuardianRequest | undefined} */
160
234
  let request
235
+ let ownerUpdate = false
161
236
 
162
237
  try {
163
238
  request = /** @type {GuardianRequest} */ (JSON.parse(line))
164
239
  if (request.token !== token) throw new Error("Guardian authentication failed")
240
+ if (ownerClient === socket && request.command === "update") {
241
+ ownerUpdate = true
242
+ ownerUpdatesInFlight.set(socket, (ownerUpdatesInFlight.get(socket) ?? 0) + 1)
243
+ }
165
244
  const result = await execute(request, socket)
166
245
 
167
246
  socket.write(`${JSON.stringify({id: request.id, result})}\n`)
168
247
  } catch (error) {
169
248
  if (!socket.destroyed) socket.write(`${JSON.stringify({error: error instanceof Error ? error.message : String(error), id: request?.id})}\n`)
249
+ } finally {
250
+ if (ownerUpdate) completeOwnerUpdate(socket)
170
251
  }
171
252
  }
172
253
 
@@ -178,16 +259,24 @@ async function handleLine(socket, line) {
178
259
  async function execute(request, socket) {
179
260
  if (shuttingDown) throw new Error("Process guardian is shutting down")
180
261
 
262
+ if (request.command === "capabilities") return {daemonRecovery: 1, generationReactivation: 1}
263
+
181
264
  if (request.command === "owner-replacement-capabilities") {
182
265
  return {commands: ["commit-retired-owner-replacement"], protocol: "owner-replacement", version: 1}
183
266
  }
184
267
 
185
268
  if (request.command === "claim-owner") {
269
+ if (retirementFailed) throw new Error("Guardian owner replacement is fenced after incomplete retired-listener state transfer")
270
+ if (ownerState !== undefined && !isDeepStrictEqual(request.authority, ownerAuthority(ownerState))) {
271
+ const waitsForPreparedAuthority = Boolean(ownerClient && replacementOwnerState && isDeepStrictEqual(request.authority, replacementAuthority))
272
+
273
+ if (!waitsForPreparedAuthority) throw new Error("Owner recovery authority does not match the guardian's committed authority")
274
+ }
186
275
  if (!ownerClient) {
187
- if (ownerState !== undefined && !isDeepStrictEqual(request.authority, ownerAuthority(ownerState))) {
188
- throw new Error("Owner recovery authority does not match the guardian's committed authority")
189
- }
276
+ acceptOwnerClaim(request.ownerPid)
190
277
  ownerClient = socket
278
+ ownerClientPid = request.ownerPid
279
+ reattachRetiredListenerSuccessors(socket)
191
280
  return {claimed: true}
192
281
  }
193
282
  if (ownerClient === socket) return {claimed: true}
@@ -199,7 +288,7 @@ async function execute(request, socket) {
199
288
  reject(new Error("Durable owner is already claimed by another matching daemon"))
200
289
  }, request.graceMs ?? 30000)
201
290
 
202
- claimWaiters.push({reject, resolve, socket, timer})
291
+ claimWaiters.push({authority: request.authority ?? null, ownerPid: request.ownerPid, reject, resolve, socket, timer})
203
292
  })
204
293
  }
205
294
 
@@ -208,7 +297,10 @@ async function execute(request, socket) {
208
297
  if (replacementClient) throw new Error("Committed owner cannot retire while an owner replacement is prepared")
209
298
  for (const entry of processes.values()) entry.desired = false
210
299
  void Promise.allSettled([...processes.values()].map((entry) => entry.process.stop()))
300
+ clearOwnerLocalListenerSource()
301
+ rememberDisconnectedOwnerSuccessors(socket)
211
302
  ownerClient = undefined
303
+ ownerClientPid = undefined
212
304
  ownerMutationClient = undefined
213
305
  ownerMutationId = undefined
214
306
  ownerRevision += 1
@@ -216,6 +308,34 @@ async function execute(request, socket) {
216
308
  return {retired: true}
217
309
  }
218
310
 
311
+ if (request.command === "owner-ready") {
312
+ requireOwner(socket, request.command)
313
+ if (!request.ownerPid || request.ownerPid !== ownerClientPid) throw new Error("Guardian owner readiness PID does not match the claimed owner")
314
+ const recovery = ownerRecoveryDefinition()
315
+ const pidPath = recovery?.command.pidPath
316
+
317
+ if (ownerRestartChild && ownerRestartChild.pid !== request.ownerPid) throw new Error("Guardian owner readiness does not match the tracked recovery child")
318
+ if (pidPath) {
319
+ let temporaryDirectory
320
+
321
+ fsSync.mkdirSync(path.dirname(pidPath), {recursive: true})
322
+ try {
323
+ temporaryDirectory = fsSync.mkdtempSync(path.join(path.dirname(pidPath), `.${path.basename(pidPath)}.`))
324
+ const temporaryPath = path.join(temporaryDirectory, "pid")
325
+
326
+ fsSync.writeFileSync(temporaryPath, `${request.ownerPid}\n`, {flag: "wx"})
327
+ fsSync.renameSync(temporaryPath, pidPath)
328
+ } finally {
329
+ if (temporaryDirectory) fsSync.rmSync(temporaryDirectory, {force: true, recursive: true})
330
+ }
331
+ }
332
+ if (ownerRestartChild) {
333
+ clearOwnerRestartTracking(ownerRestartChild)
334
+ }
335
+ ownerRestartRetryDelayMs = undefined
336
+ return {ready: true}
337
+ }
338
+
219
339
  if (request.command === "abandon-legacy-upgrade") {
220
340
  if (!legacyGuardian) throw new Error("Guardian is not a legacy upgrade coordinator")
221
341
  if (ownerClient || committedReplacementId) throw new Error("Committed guardian authority cannot abandon its legacy backend")
@@ -231,7 +351,13 @@ async function execute(request, socket) {
231
351
  if (!legacyGuardian) throw new Error("Guardian is not a legacy upgrade coordinator")
232
352
  requireReplacement(socket, request)
233
353
  if (legacyOwnerClaim) throw new Error("Legacy guardian owner claim is already pending")
234
- legacyOwnerClaim = legacyGuardian.claimOwner(request.graceMs ?? 30000, ownerAuthority(ownerState)).then(
354
+ if (replacementAuthority === undefined) throw new Error("Legacy guardian owner claim is missing replacement authority")
355
+ if (!request.statePath || !path.isAbsolute(request.statePath)) throw new Error("Legacy guardian owner claim requires an absolute state path")
356
+ if (request.statePath !== ownerStatePath(ownerState)) throw new Error("Legacy guardian owner claim state path does not match the committed owner config")
357
+ assertBridgeRecoverySnapshot(request.recoverySnapshot)
358
+ legacyRecoveryStatePath = request.statePath
359
+ legacyRecoverySnapshot = request.recoverySnapshot
360
+ legacyOwnerClaim = legacyGuardian.claimOwner(request.graceMs ?? 30000, replacementAuthority).then(
235
361
  () => undefined,
236
362
  (error) => error instanceof Error ? error : new Error(String(error))
237
363
  )
@@ -244,15 +370,17 @@ async function execute(request, socket) {
244
370
  const claimError = await legacyOwnerClaim
245
371
 
246
372
  if (claimError) throw claimError
373
+ if (!legacyRecoveryStatePath || legacyRecoverySnapshot === undefined) throw new Error("Legacy guardian owner claim is missing durable recovery state")
374
+ await writeState(legacyRecoveryStatePath, legacyRecoverySnapshot)
247
375
  return {claimed: true}
248
376
  }
249
377
 
250
378
  if (request.command === "publish-owner-state") {
251
379
  requireOwner(socket, request.command)
252
380
  if (request.ownerState === undefined) throw new Error("Guardian owner publication requires ownerState")
253
- ownerState = request.ownerState
381
+ ownerState = mergeOwnerConnectionState(ownerState, request.ownerState)
254
382
  ownerRevision += 1
255
- committedReplacementId = undefined
383
+ if (!retiringClient) committedReplacementId = undefined
256
384
  return {published: true}
257
385
  }
258
386
 
@@ -263,7 +391,7 @@ async function execute(request, socket) {
263
391
  }
264
392
 
265
393
  if (request.command === "replacement-status") {
266
- return {committedReplacementId: committedReplacementId ?? null, ownerClaimed: Boolean(ownerClient), retirementPending: Boolean(retiringClient)}
394
+ return {committedReplacementId: committedReplacementId ?? null, ownerClaimed: Boolean(ownerClient), retirementFailed, retirementPending: Boolean(retiringClient), retirementReady: retiringListenerReady}
267
395
  }
268
396
 
269
397
  if (request.command === "begin-owner-mutation") {
@@ -284,6 +412,8 @@ async function execute(request, socket) {
284
412
  }
285
413
 
286
414
  if (request.command === "prepare-owner-replacement") {
415
+ if (retirementFailed) throw new Error("Guardian owner replacement is fenced after incomplete retired-listener state transfer")
416
+ if (retiringClient) throw new Error("Guardian owner replacement cannot prepare while listener retirement is pending")
287
417
  if (socket === ownerClient) throw new Error("Committed owner cannot prepare itself as a replacement")
288
418
  if (replacementClient && replacementClient !== socket) throw new Error("Another owner replacement candidate is already prepared")
289
419
  if (ownerMutationClient) throw new Error("Owner replacement cannot prepare while an owner mutation is in progress")
@@ -295,8 +425,18 @@ async function execute(request, socket) {
295
425
  throw new Error("Owner replacement authority fence does not match the guardian's committed authority")
296
426
  }
297
427
  if (request.nextAuthority === undefined) throw new Error("Owner replacement requires the requested authority")
428
+ if (!ownerClient) {
429
+ cancelOwnerRestart()
430
+ const supersededRestartChild = ownerRestartChild
431
+
432
+ clearOwnerRestartTracking()
433
+ if (supersededRestartChild) killDetachedProcessGroup(supersededRestartChild)
434
+ }
298
435
  if (!replacementId) replacementId = crypto.randomUUID()
436
+ retirementFailed = false
437
+ retiringListenerReady = false
299
438
  replacementClient = socket
439
+ replacementOwnerPid = request.ownerPid
300
440
  replacementAuthority = request.nextAuthority
301
441
  replacementRevision = ownerRevision
302
442
  if (ownerClient && !ownerClient.destroyed) ownerClient.write(`${JSON.stringify({event: "replacement-prepared", replacementId})}\n`)
@@ -309,7 +449,7 @@ async function execute(request, socket) {
309
449
  if (request.ownerState === undefined || !isDeepStrictEqual(ownerAuthority(request.ownerState), replacementAuthority)) {
310
450
  throw new Error("Prepared replacement state does not match the requested authority")
311
451
  }
312
- replacementOwnerState = request.ownerState
452
+ replacementOwnerState = mergeOwnerConnectionState(ownerState, request.ownerState)
313
453
  if (!ownerClient) commitReplacement()
314
454
  return {committed: ownerClient === socket}
315
455
  }
@@ -320,30 +460,94 @@ async function execute(request, socket) {
320
460
  return {aborted: true}
321
461
  }
322
462
 
463
+ if (request.command === "prepare-retired-owner-listener-handoff") {
464
+ requireReplacement(socket, request)
465
+ requireRetiredOwnerReplacement(request)
466
+ await requireMissingOwnerControlPath(ownerState)
467
+ requireReplacement(socket, request)
468
+ if (!ownerClient || ownerClient.destroyed) throw new Error("Retired owner listener handoff requires the connected committed owner")
469
+ if (retiringClient) throw new Error("Another owner listener retirement is already pending")
470
+ retiringClient = ownerClient
471
+ retiringReplacementId = /** @type {string} */ (request.replacementId)
472
+ retiredListenerClients.set(ownerClient, {
473
+ listenerStateComplete: false,
474
+ localSources: new Map(),
475
+ replacementId: /** @type {string} */ (request.replacementId),
476
+ successor: socket
477
+ })
478
+ ownerClient.write(`${JSON.stringify({event: "replacement-listener-handoff-requested", replacementId: request.replacementId})}\n`)
479
+ return {prepared: true}
480
+ }
481
+
323
482
  if (request.command === "commit-retired-owner-replacement") {
324
483
  requireReplacement(socket, request)
325
- requireProcess(request)
326
- if (!committedOwnerProcessKeys(ownerState).has(/** @type {string} */ (request.key))) {
327
- throw new Error(`Guardian process ${request.key} does not belong to the committed owner`)
328
- }
329
- if (!replacementOwnerState) throw new Error("Retired owner replacement transaction is not staged")
330
- if (!isDeepStrictEqual(ownerAuthority(ownerState), replacementAuthority)) throw new Error("Retired owner replacement requires unchanged owner authority")
331
- const controlPath = ownerControlPath(ownerState)
484
+ requireRetiredOwnerReplacement(request)
485
+ await requireMissingOwnerControlPath(ownerState)
486
+ const listener = retiringClient
332
487
 
333
- try {
334
- await fs.lstat(controlPath)
335
- throw new Error(`Retired owner control socket ${controlPath} still exists`)
336
- } catch (error) {
337
- if (!error || typeof error !== "object" || !("code" in error) || error.code !== "ENOENT") throw error
488
+ if (!listener || listener !== ownerClient || listener.destroyed || !retiringListenerReady || retiredListenerClients.get(listener)?.successor !== socket) {
489
+ throw new Error("Retired owner replacement requires a complete prepared listener handoff")
338
490
  }
339
491
  commitReplacement()
340
- finalizeReplacementRetirement()
492
+ if (ownerClient) {
493
+ const retirement = retiredListenerClients.get(listener)
494
+
495
+ if (!retirement) throw new Error("Retired owner replacement lost its prepared listener handoff")
496
+ retirement.successor = ownerClient
497
+ listener.write(`${JSON.stringify({event: "replacement-retirement-requested", replacementId: request.replacementId})}\n`)
498
+ }
341
499
  return {committed: true}
342
500
  }
343
501
 
502
+ if (request.command === "publish-owner-connection-state") {
503
+ const retirement = retiredListenerClients.get(socket)
504
+
505
+ if (!retirement || retirement.replacementId !== request.replacementId) throw new Error("Owner connection state requires the exact retired listener transaction")
506
+ if (typeof request.sourceId !== "string" || !request.sourceId || typeof request.releaseId !== "string" || !request.connections || typeof request.connections !== "object" || Array.isArray(request.connections)) {
507
+ throw new Error("Owner connection state requires a source, release, and connection counts")
508
+ }
509
+ const connections = request.connections
510
+ const http = connections.http
511
+ const websocket = connections.websocket
512
+
513
+ if (typeof http !== "number" || !Number.isSafeInteger(http) || http < 0 || typeof websocket !== "number" || !Number.isSafeInteger(websocket) || websocket < 0) {
514
+ throw new Error("Owner connection state requires non-negative integer counts")
515
+ }
516
+ const normalized = {http, websocket}
517
+
518
+ if (request.localSource) setRetiredLocalSource(retirement, request.sourceId, request.releaseId, normalized)
519
+ applyOwnerConnectionState(ownerState, request.sourceId, request.releaseId, normalized)
520
+ if (replacementOwnerState) applyOwnerConnectionState(replacementOwnerState, request.sourceId, request.releaseId, normalized)
521
+ publishOwnerConnectionState(retirement.successor, request.sourceId, request.releaseId, normalized)
522
+ return {published: true}
523
+ }
524
+
525
+ if (request.command === "complete-owner-listener-retirement") {
526
+ if (retiredListenerClients.get(socket)?.replacementId !== request.replacementId || retiringClient !== socket) {
527
+ throw new Error("Owner listener retirement completion requires the exact retired listener transaction")
528
+ }
529
+ retiringListenerReady = true
530
+ const retirement = retiredListenerClients.get(socket)
531
+
532
+ if (!retirement) throw new Error("Owner listener retirement completion lost its transaction")
533
+ retirement.listenerStateComplete = true
534
+ const successor = retirement.successor
535
+
536
+ if (successor && !successor.destroyed) {
537
+ successor.write(`${JSON.stringify({event: "replacement-listeners-retired", replacementId: request.replacementId})}\n`)
538
+ }
539
+ return {completed: true}
540
+ }
541
+
344
542
  if (request.command === "commit-owner-replacement") {
345
543
  requireOwner(socket, request.command)
346
544
  if (!replacementClient || request.replacementId !== replacementId || !replacementOwnerState) throw new Error("Owner replacement transaction is not the prepared ready candidate")
545
+ retiredListenerClients.set(socket, {
546
+ listenerStateComplete: false,
547
+ localSources: new Map(),
548
+ replacementId: /** @type {string} */ (request.replacementId),
549
+ successor: replacementClient
550
+ })
347
551
  commitReplacement()
348
552
  return {committed: true}
349
553
  }
@@ -355,7 +559,15 @@ async function execute(request, socket) {
355
559
  }
356
560
 
357
561
  if (request.command === "finalize-owner-replacement") {
358
- if (retiringClient !== socket || request.replacementId !== retiringReplacementId) throw new Error("Owner replacement retirement finalization requires the committed incumbent transaction")
562
+ const incumbentRetirement = retiredListenerClients.get(socket)
563
+ const incumbentFinalization = retiringClient === socket && (!incumbentRetirement || incumbentRetirement.listenerStateComplete)
564
+ const candidateFinalization = ownerClient === socket && retiringListenerReady && !retirementFailed
565
+ const alreadyFinalized = ownerClient === socket && !retiringClient && request.replacementId === committedReplacementId && !retirementFailed
566
+
567
+ if (alreadyFinalized) return {finalized: true}
568
+ if ((!incumbentFinalization && !candidateFinalization) || request.replacementId !== retiringReplacementId) {
569
+ throw new Error("Owner replacement retirement finalization requires the committed listener transaction")
570
+ }
359
571
  finalizeReplacementRetirement()
360
572
  return {finalized: true}
361
573
  }
@@ -424,7 +636,7 @@ async function execute(request, socket) {
424
636
  : new ManagedProcess(managedDefinition)
425
637
 
426
638
  managedProcess.on("log", (entry) => {
427
- broadcast({entry, event: "process-log", key: request.key, status: managedProcess.status()})
639
+ broadcast({entry, event: "process-log", key: request.key})
428
640
  })
429
641
  if (recoversLegacyProcess && "recover" in managedProcess && typeof managedProcess.recover === "function") await managedProcess.recover()
430
642
 
@@ -443,6 +655,9 @@ async function execute(request, socket) {
443
655
  await record.process.start(request.reason, request.lifecycleRole)
444
656
  } else if (request.command === "activate") {
445
657
  await record.process.activateStrict()
658
+ } else if (request.command === "reactivate" || request.command === "reactivate-with-command") {
659
+ await record.process.reactivateStrict()
660
+ record.desired = true
446
661
  } else if (request.command === "quiesce") {
447
662
  record.desired = false
448
663
  await record.process.quiesceStrict()
@@ -458,7 +673,7 @@ async function execute(request, socket) {
458
673
  } else if (request.command === "update") {
459
674
  if (!request.definition || !request.provenance) throw new Error("Guardian update requires definition and provenance")
460
675
  if (record.provenance !== request.previousProvenance) throw new Error(`Guardian provenance mismatch for ${request.key}`)
461
- record.process.updateDefinition({
676
+ await record.process.updateDefinition({
462
677
  ...request.definition,
463
678
  lifecycle: request.definition.lifecycle || {drainTimeoutMs: 0},
464
679
  logger: record.process.logger,
@@ -468,6 +683,10 @@ async function execute(request, socket) {
468
683
  stopSignal: request.definition.stopSignal || "SIGTERM"
469
684
  })
470
685
  record.provenance = request.provenance
686
+ if (request.ownerState !== undefined) {
687
+ ownerState = mergeOwnerConnectionState(ownerState, request.ownerState)
688
+ committedReplacementId = undefined
689
+ }
471
690
  } else if (request.command === "status") {
472
691
  return record.process.status()
473
692
  } else {
@@ -490,17 +709,83 @@ function requireOwner(socket, command) {
490
709
  if (ownerClient !== socket) throw new Error(`Guardian ${command} requires the committed owner`)
491
710
  }
492
711
 
493
- /** @param {string} reason - Abort diagnostic. */
494
- function abortReplacement(reason) {
495
- const event = `${JSON.stringify({event: "replacement-aborted", reason})}\n`
712
+ /**
713
+ * Releases authority only after every accepted request from the disconnected owner settles.
714
+ * @param {net.Socket} socket - Disconnected owner socket.
715
+ */
716
+ function releaseClosedOwner(socket) {
717
+ if (ownerClient !== socket) return
718
+ const incompleteRestartChild = ownerRestartChild?.pid === ownerClientPid ? ownerRestartChild : undefined
719
+
720
+ clearOwnerLocalListenerSource()
721
+ rememberDisconnectedOwnerSuccessors(socket)
722
+ ownerClient = undefined
723
+ ownerClientPid = undefined
724
+ if (ownerMutationClient === socket) {
725
+ ownerMutationClient = undefined
726
+ ownerMutationId = undefined
727
+ }
728
+ if (replacementClient && replacementOwnerState) commitReplacement()
729
+ else {
730
+ if (incompleteRestartChild) {
731
+ reportOwnerRestartFailure(ownerRecoveryDefinition(), {code: "OWNER_DISCONNECTED"})
732
+ killDetachedProcessGroup(incompleteRestartChild)
733
+ clearOwnerRestartTracking(incompleteRestartChild)
734
+ ownerRestartRetryDelayMs = OWNER_RESTART_RETRY_MS
735
+ }
736
+ if (replacementClient) abortReplacement("Committed owner disconnected before replacement candidate was ready", false)
737
+ if (retirementFailed) return
738
+ if (!shuttingDown) {
739
+ grantNextOwner()
740
+ const retryDelayMs = ownerRestartRetryDelayMs
741
+
742
+ ownerRestartRetryDelayMs = undefined
743
+ scheduleOwnerRestart(retryDelayMs)
744
+ }
745
+ }
746
+ }
747
+
748
+ /**
749
+ * Completes one accepted owner update and applies a deferred disconnect boundary.
750
+ * @param {net.Socket} socket - Requesting owner socket.
751
+ */
752
+ function completeOwnerUpdate(socket) {
753
+ const remaining = (ownerUpdatesInFlight.get(socket) ?? 1) - 1
754
+
755
+ if (remaining > 0) {
756
+ ownerUpdatesInFlight.set(socket, remaining)
757
+ return
758
+ }
759
+ ownerUpdatesInFlight.delete(socket)
760
+ if (socket.destroyed) releaseClosedOwner(socket)
761
+ }
496
762
 
497
- if (replacementClient && !replacementClient.destroyed) replacementClient.write(event)
498
- if (ownerClient && !ownerClient.destroyed) ownerClient.write(event)
763
+ /**
764
+ * @param {string} reason - Abort diagnostic.
765
+ * @param {boolean} [scheduleRecovery] - Whether this call owns recovery scheduling.
766
+ */
767
+ function abortReplacement(reason, scheduleRecovery = true) {
768
+ const abortedReplacementId = replacementId
769
+
770
+ if (replacementClient && !replacementClient.destroyed) replacementClient.write(`${JSON.stringify({event: "replacement-aborted", reason})}\n`)
771
+ if (ownerClient && !ownerClient.destroyed) ownerClient.write(`${JSON.stringify({event: "replacement-aborted", reason})}\n`)
772
+ if (retiringClient && retiringClient === ownerClient && retiringReplacementId === abortedReplacementId) {
773
+ const retirement = retiredListenerClients.get(retiringClient)
774
+
775
+ if (retirement) disconnectedOwnerSuccessors.delete(retirement)
776
+ retiredListenerClients.delete(retiringClient)
777
+ retiringClient = undefined
778
+ retiringReplacementId = undefined
779
+ retiringListenerReady = false
780
+ retirementFailed = false
781
+ }
499
782
  replacementClient = undefined
500
783
  replacementId = undefined
501
784
  replacementAuthority = undefined
502
785
  replacementOwnerState = undefined
786
+ replacementOwnerPid = undefined
503
787
  replacementRevision = undefined
788
+ if (scheduleRecovery && !ownerClient) scheduleOwnerRestart()
504
789
  }
505
790
 
506
791
  /** Atomically promotes the prepared client and its complete transferable state. */
@@ -510,14 +795,31 @@ function commitReplacement() {
510
795
  const committedClient = replacementClient
511
796
  const committedId = replacementId
512
797
 
798
+ cancelOwnerRestart()
799
+ const supersededRestartChild = ownerRestartChild
800
+ const restartChildIsIncumbent = Boolean(previousOwner && supersededRestartChild?.pid === ownerClientPid)
801
+
802
+ clearOwnerRestartTracking()
803
+ if (supersededRestartChild && !restartChildIsIncumbent) killDetachedProcessGroup(supersededRestartChild)
513
804
  ownerClient = committedClient
805
+ ownerClientPid = replacementOwnerPid
806
+ ownerRestartRetryDelayMs = undefined
514
807
  ownerState = replacementOwnerState
515
808
  ownerRevision += 1
516
809
  committedReplacementId = committedId
810
+ for (const waiter of claimWaiters.splice(0)) {
811
+ if (isDeepStrictEqual(waiter.authority, ownerAuthority(ownerState))) {
812
+ claimWaiters.push(waiter)
813
+ } else {
814
+ clearTimeout(waiter.timer)
815
+ waiter.reject(new Error("Durable owner authority changed while the claim was queued"))
816
+ }
817
+ }
517
818
  replacementClient = undefined
518
819
  replacementId = undefined
519
820
  replacementAuthority = undefined
520
821
  replacementOwnerState = undefined
822
+ replacementOwnerPid = undefined
521
823
  replacementRevision = undefined
522
824
  if (previousOwner && !previousOwner.destroyed) {
523
825
  retiringClient = previousOwner
@@ -535,6 +837,8 @@ function finalizeReplacementRetirement() {
535
837
 
536
838
  retiringClient = undefined
537
839
  retiringReplacementId = undefined
840
+ retiringListenerReady = false
841
+ retirementFailed = false
538
842
  if (!committedClient || !committedId) return
539
843
  publishReplacementCommitted(committedClient, committedId)
540
844
  if (previousOwner && !previousOwner.destroyed) previousOwner.write(`${JSON.stringify({event: "replacement-retired", replacementId: committedId})}\n`)
@@ -568,6 +872,144 @@ function requireProcess(request) {
568
872
  return record
569
873
  }
570
874
 
875
+ /**
876
+ * @param {RetiredListenerTransaction} retirement - Retired listener transaction.
877
+ * @param {string} sourceId - Physically-owned source.
878
+ * @param {string} releaseId - Retained release.
879
+ * @param {{http: number, websocket: number}} connections - Exact live counts.
880
+ */
881
+ function setRetiredLocalSource(retirement, sourceId, releaseId, connections) {
882
+ const releases = retirement.localSources.get(sourceId) || new Map()
883
+
884
+ if (connections.http + connections.websocket === 0) releases.delete(releaseId)
885
+ else releases.set(releaseId, connections)
886
+ if (releases.size === 0) retirement.localSources.delete(sourceId)
887
+ else retirement.localSources.set(sourceId, releases)
888
+ }
889
+
890
+ /**
891
+ * @param {net.Socket | undefined} firstSuccessor - First daemon that inherited the source.
892
+ * @param {string} sourceId - Stable listener source.
893
+ * @param {string} releaseId - Retained release.
894
+ * @param {{http: number, websocket: number}} connections - Exact live counts.
895
+ * @param {Set<net.Socket>} [delivered] - Successors that already received this update.
896
+ */
897
+ function publishOwnerConnectionState(firstSuccessor, sourceId, releaseId, connections, delivered = new Set()) {
898
+ let successor = firstSuccessor
899
+
900
+ while (successor && !delivered.has(successor)) {
901
+ delivered.add(successor)
902
+ if (!successor.destroyed) {
903
+ successor.write(`${JSON.stringify({connections, event: "owner-connection-state", releaseId, sourceId})}\n`)
904
+ }
905
+ successor = retiredListenerClients.get(successor)?.successor
906
+ }
907
+ }
908
+
909
+ /** @param {RetiredListenerTransaction} retirement - Disconnected completed listener. */
910
+ function publishClosedLocalSources(retirement) {
911
+ for (const [sourceId, releases] of retirement.localSources) {
912
+ for (const releaseId of releases.keys()) {
913
+ const connections = {http: 0, websocket: 0}
914
+ const delivered = /** @type {Set<net.Socket>} */ (new Set())
915
+
916
+ applyOwnerConnectionState(ownerState, sourceId, releaseId, connections)
917
+ if (replacementOwnerState) {
918
+ applyOwnerConnectionState(replacementOwnerState, sourceId, releaseId, connections)
919
+ publishOwnerConnectionState(replacementClient, sourceId, releaseId, connections, delivered)
920
+ }
921
+ publishOwnerConnectionState(retirement.successor, sourceId, releaseId, connections, delivered)
922
+ }
923
+ }
924
+ }
925
+
926
+ /**
927
+ * @param {import("./json.js").JsonValue | undefined} state - Candidate or committed private owner state.
928
+ * @param {string} sourceId - Stable listener source.
929
+ * @param {string} releaseId - Retained release.
930
+ * @param {{http: number, websocket: number}} connections - Exact live counts.
931
+ */
932
+ function applyOwnerConnectionState(state, sourceId, releaseId, connections) {
933
+ if (!state || typeof state !== "object" || Array.isArray(state)) throw new Error("Guardian owner connection state requires transferable owner state")
934
+ const transferable = /** @type {{listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>}} */ (state)
935
+ const sources = transferable.listenerConnectionSources || {}
936
+ const releases = sources[sourceId] || {}
937
+
938
+ if (connections.http + connections.websocket === 0) {
939
+ delete releases[releaseId]
940
+ } else {
941
+ releases[releaseId] = connections
942
+ }
943
+ if (Object.keys(releases).length === 0) delete sources[sourceId]
944
+ else sources[sourceId] = releases
945
+ transferable.listenerConnectionSources = sources
946
+ }
947
+
948
+ /** Clears the physically-owned source before another daemon reconstructs committed state. */
949
+ function clearOwnerLocalListenerSource() {
950
+ if (!ownerState || typeof ownerState !== "object" || Array.isArray(ownerState)) return
951
+ const transferable = /** @type {{listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>, listenerSourceId?: string}} */ (ownerState)
952
+ const sourceId = transferable.listenerSourceId
953
+
954
+ if (!sourceId) return
955
+ for (const releaseId of Object.keys(transferable.listenerConnectionSources?.[sourceId] || {})) {
956
+ const connections = {http: 0, websocket: 0}
957
+
958
+ applyOwnerConnectionState(ownerState, sourceId, releaseId, connections)
959
+ if (replacementOwnerState) {
960
+ applyOwnerConnectionState(replacementOwnerState, sourceId, releaseId, connections)
961
+ publishOwnerConnectionState(replacementClient, sourceId, releaseId, connections)
962
+ }
963
+ }
964
+ }
965
+
966
+ /**
967
+ * Preserves guardian-confirmed source updates while accepting the publisher's own local source.
968
+ * @param {import("./json.js").JsonValue | undefined} previousState - Current guardian state.
969
+ * @param {import("./json.js").JsonValue} nextState - Incoming owner publication.
970
+ * @returns {import("./json.js").JsonValue} Publication with guardian source state merged in.
971
+ */
972
+ function mergeOwnerConnectionState(previousState, nextState) {
973
+ if (!nextState || typeof nextState !== "object" || Array.isArray(nextState)) throw new Error("Guardian owner publication requires transferable owner state")
974
+ const next = /** @type {{listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>, listenerSourceId?: string}} */ (nextState)
975
+ const publishesSources = next.listenerConnectionSources !== undefined
976
+ const localSourceId = next.listenerSourceId
977
+
978
+ if (!previousState || typeof previousState !== "object" || Array.isArray(previousState) || !localSourceId) return nextState
979
+ const previous = /** @type {{listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>}} */ (previousState)
980
+ const sources = /** @type {Record<string, Record<string, {http: number, websocket: number}>>} */ ({})
981
+ const localReleases = next.listenerConnectionSources?.[localSourceId]
982
+
983
+ if (localReleases && Object.keys(localReleases).length > 0) sources[localSourceId] = localReleases
984
+ for (const [sourceId, releases] of Object.entries(previous.listenerConnectionSources || {})) {
985
+ if (sourceId !== localSourceId) sources[sourceId] = releases
986
+ }
987
+ if (publishesSources || Object.keys(sources).length > 0) next.listenerConnectionSources = sources
988
+ return nextState
989
+ }
990
+
991
+ /** @param {GuardianRequest} request - Staged same-authority listener handoff request. */
992
+ function requireRetiredOwnerReplacement(request) {
993
+ requireProcess(request)
994
+ if (!committedOwnerProcessKeys(ownerState).has(/** @type {string} */ (request.key))) {
995
+ throw new Error(`Guardian process ${request.key} does not belong to the committed owner`)
996
+ }
997
+ if (!replacementOwnerState) throw new Error("Retired owner replacement transaction is not staged")
998
+ if (!isDeepStrictEqual(ownerAuthority(ownerState), replacementAuthority)) throw new Error("Retired owner replacement requires unchanged owner authority")
999
+ }
1000
+
1001
+ /** @param {import("./json.js").JsonValue | undefined} state - Committed transferable state. */
1002
+ async function requireMissingOwnerControlPath(state) {
1003
+ const controlPath = ownerControlPath(state)
1004
+
1005
+ try {
1006
+ await fs.lstat(controlPath)
1007
+ throw new Error(`Retired owner control socket ${controlPath} still exists`)
1008
+ } catch (error) {
1009
+ if (!error || typeof error !== "object" || !("code" in error) || error.code !== "ENOENT") throw error
1010
+ }
1011
+ }
1012
+
571
1013
  /**
572
1014
  * @param {import("./json.js").JsonValue} state - Transfer state.
573
1015
  * @returns {import("./json.js").JsonValue} Embedded authority fence.
@@ -592,6 +1034,31 @@ function ownerControlPath(state) {
592
1034
  return control.path
593
1035
  }
594
1036
 
1037
+ /**
1038
+ * @param {import("./json.js").JsonValue | undefined} state - Committed transferable state.
1039
+ * @returns {string} Exact configured public state path.
1040
+ */
1041
+ function ownerStatePath(state) {
1042
+ if (!state || typeof state !== "object" || Array.isArray(state) || !("config" in state)) throw new Error("Guardian owner state is missing its committed config")
1043
+ const config = state.config
1044
+
1045
+ if (!config || typeof config !== "object" || Array.isArray(config) || !("statePath" in config) || typeof config.statePath !== "string") throw new Error("Guardian owner config is missing its state path")
1046
+ return config.statePath
1047
+ }
1048
+
1049
+ /** @param {import("./json.js").JsonValue | undefined} snapshot - Candidate public recovery snapshot. */
1050
+ function assertBridgeRecoverySnapshot(snapshot) {
1051
+ if (!snapshot || typeof snapshot !== "object" || Array.isArray(snapshot) || !("recovery" in snapshot)) throw new Error("Legacy guardian owner claim requires a recovery snapshot")
1052
+ const recovery = snapshot.recovery
1053
+
1054
+ if (!recovery || typeof recovery !== "object" || Array.isArray(recovery) || !("guardian" in recovery)) throw new Error("Legacy guardian owner claim recovery snapshot is missing its guardian")
1055
+ const guardian = recovery.guardian
1056
+
1057
+ if (!guardian || typeof guardian !== "object" || Array.isArray(guardian) || guardian.socketPath !== socketPath || guardian.token !== token) {
1058
+ throw new Error("Legacy guardian owner claim recovery snapshot does not identify this guardian")
1059
+ }
1060
+ }
1061
+
595
1062
  /**
596
1063
  * Stops accepting connections and closes every authority channel except the response caller.
597
1064
  * @param {net.Socket} caller - Shutdown requester retained until it receives the response.
@@ -624,16 +1091,75 @@ async function finishShutdown() {
624
1091
  function broadcast(event) {
625
1092
  const line = `${JSON.stringify(event)}\n`
626
1093
 
627
- for (const client of clients) if (!client.destroyed) client.write(line)
1094
+ for (const client of clients) {
1095
+ if (client.destroyed) continue
1096
+ if (backpressuredClients.has(client)) {
1097
+ if (typeof event.key === "string") {
1098
+ const pending = pendingStatusEvents.get(client) || new Map()
1099
+
1100
+ if (event.event === "process-log") {
1101
+ const record = processes.get(event.key)
1102
+
1103
+ if (record) pending.set(event.key, `${JSON.stringify({event: "status", key: event.key, status: record.process.status()})}\n`)
1104
+ } else {
1105
+ pending.set(event.key, line)
1106
+ }
1107
+ pendingStatusEvents.set(client, pending)
1108
+ }
1109
+ continue
1110
+ }
1111
+ if (!client.write(line)) backpressuredClients.add(client)
1112
+ }
1113
+ }
1114
+
1115
+ /**
1116
+ * Flushes one latest status-bearing event per process after socket backpressure clears.
1117
+ * @param {net.Socket} client - Drained guardian client socket.
1118
+ */
1119
+ function flushStatusEvents(client) {
1120
+ backpressuredClients.delete(client)
1121
+ const pending = pendingStatusEvents.get(client)
1122
+
1123
+ if (!pending || client.destroyed) return
1124
+ for (const [key, line] of pending) {
1125
+ pending.delete(key)
1126
+ if (!client.write(line)) {
1127
+ backpressuredClients.add(client)
1128
+ break
1129
+ }
1130
+ }
1131
+ if (pending.size === 0) pendingStatusEvents.delete(client)
1132
+ }
1133
+
1134
+ /** @param {net.Socket} owner - Disconnected or retiring committed owner. */
1135
+ function rememberDisconnectedOwnerSuccessors(owner) {
1136
+ for (const transaction of retiredListenerClients.values()) {
1137
+ if (transaction.successor === owner) disconnectedOwnerSuccessors.add(transaction)
1138
+ }
1139
+ }
1140
+
1141
+ /** @param {net.Socket} successor - Newly claimed recovery owner. */
1142
+ function reattachRetiredListenerSuccessors(successor) {
1143
+ for (const transaction of disconnectedOwnerSuccessors) transaction.successor = successor
1144
+ disconnectedOwnerSuccessors.clear()
628
1145
  }
629
1146
 
630
1147
  /** @returns {void} Grants the next queued owner claim. */
631
1148
  function grantNextOwner() {
632
- const next = claimWaiters.shift()
1149
+ let next = claimWaiters.shift()
1150
+
1151
+ while (next && ownerState !== undefined && !isDeepStrictEqual(next.authority, ownerAuthority(ownerState))) {
1152
+ clearTimeout(next.timer)
1153
+ next.reject(new Error("Owner recovery authority changed while the claim was queued"))
1154
+ next = claimWaiters.shift()
1155
+ }
633
1156
 
634
1157
  if (!next) return
635
1158
  clearTimeout(next.timer)
1159
+ acceptOwnerClaim(next.ownerPid)
636
1160
  ownerClient = next.socket
1161
+ ownerClientPid = next.ownerPid
1162
+ reattachRetiredListenerSuccessors(next.socket)
637
1163
  next.resolve({claimed: true})
638
1164
  for (const waiter of claimWaiters.splice(0)) {
639
1165
  clearTimeout(waiter.timer)
@@ -641,6 +1167,171 @@ function grantNextOwner() {
641
1167
  }
642
1168
  }
643
1169
 
1170
+ /** @param {number} [delayMs] - Explicit retry delay, otherwise the accepted reconnect grace. */
1171
+ function scheduleOwnerRestart(delayMs) {
1172
+ if (shuttingDown || ownerClient || replacementClient || ownerRestartChild || ownerRestartTimer) return
1173
+ const recovery = ownerRecoveryDefinition()
1174
+
1175
+ if (!recovery) return
1176
+ ownerRestartTimer = setTimeout(() => {
1177
+ ownerRestartTimer = undefined
1178
+ if (shuttingDown || ownerClient || replacementClient || ownerRestartChild) return
1179
+ const currentRecovery = ownerRecoveryDefinition()
1180
+
1181
+ if (!currentRecovery) return
1182
+ let stdoutFd
1183
+ let stderrFd
1184
+ let child
1185
+
1186
+ try {
1187
+ if (currentRecovery.command.logPath) {
1188
+ fsSync.mkdirSync(path.dirname(currentRecovery.command.logPath), {recursive: true})
1189
+ stdoutFd = fsSync.openSync(currentRecovery.command.logPath, "a")
1190
+ stderrFd = fsSync.openSync(currentRecovery.command.logPath, "a")
1191
+ }
1192
+ child = spawn(currentRecovery.command.executable, currentRecovery.command.args, {
1193
+ cwd: currentRecovery.command.cwd,
1194
+ detached: true,
1195
+ env: currentRecovery.command.env,
1196
+ stdio: currentRecovery.command.logPath ? ["ignore", stdoutFd, stderrFd] : ["ignore", "inherit", "inherit"]
1197
+ })
1198
+ } catch (error) {
1199
+ reportOwnerRestartFailure(currentRecovery, restartFailure(error instanceof Error ? error : String(error)))
1200
+ scheduleOwnerRestart(OWNER_RESTART_RETRY_MS)
1201
+ return
1202
+ } finally {
1203
+ if (stdoutFd !== undefined) fsSync.closeSync(stdoutFd)
1204
+ if (stderrFd !== undefined) fsSync.closeSync(stderrFd)
1205
+ }
1206
+
1207
+ ownerRestartChild = child
1208
+ let settled = false
1209
+ const retry = (/** @type {{code: string, exitCode?: number | null, signal?: ChildExitSignal}} */ failure) => {
1210
+ if (settled || ownerRestartChild !== child) return
1211
+ settled = true
1212
+ killDetachedProcessGroup(child)
1213
+ reportOwnerRestartFailure(currentRecovery, failure)
1214
+ clearOwnerRestartTracking(child)
1215
+ ownerRestartRetryDelayMs = OWNER_RESTART_RETRY_MS
1216
+ if (ownerClient && ownerClientPid === child.pid) ownerClient.destroy()
1217
+ else {
1218
+ ownerRestartRetryDelayMs = undefined
1219
+ scheduleOwnerRestart(OWNER_RESTART_RETRY_MS)
1220
+ }
1221
+ }
1222
+
1223
+ ownerRestartStartupTimer = setTimeout(() => {
1224
+ if (ownerRestartChild !== child) return
1225
+ retry({code: "STARTUP_TIMEOUT"})
1226
+ }, currentRecovery.startupTimeoutMs)
1227
+ ownerRestartStartupTimer.unref()
1228
+ child.once("error", (error) => retry(restartFailure(error)))
1229
+ child.once("exit", (code, signal) => retry({code: "EARLY_EXIT", exitCode: code, signal}))
1230
+ child.unref()
1231
+ }, delayMs ?? recovery.reconnectGraceMs)
1232
+ ownerRestartTimer.unref()
1233
+ }
1234
+
1235
+ /** Cancels recovery for a superseded owner command. */
1236
+ function cancelOwnerRestart() {
1237
+ if (!ownerRestartTimer) return
1238
+ clearTimeout(ownerRestartTimer)
1239
+ ownerRestartTimer = undefined
1240
+ }
1241
+
1242
+ /**
1243
+ * Accepts an exact claimant and retires any different in-flight recovery child.
1244
+ * @param {number | undefined} ownerPid - Exact claimant process PID.
1245
+ */
1246
+ function acceptOwnerClaim(ownerPid) {
1247
+ cancelOwnerRestart()
1248
+ const child = ownerRestartChild
1249
+
1250
+ if (!child || child.pid === ownerPid) return
1251
+ clearOwnerRestartTracking(child)
1252
+ killDetachedProcessGroup(child)
1253
+ }
1254
+
1255
+ /**
1256
+ * Clears recovery-child startup tracking without disturbing a committed retired listener owner.
1257
+ * @param {import("node:child_process").ChildProcess} [expectedChild] - Optional exact child fence.
1258
+ * @returns {boolean} Whether matching tracking was cleared.
1259
+ */
1260
+ function clearOwnerRestartTracking(expectedChild) {
1261
+ if (expectedChild && ownerRestartChild !== expectedChild) return false
1262
+ if (ownerRestartStartupTimer) clearTimeout(ownerRestartStartupTimer)
1263
+ ownerRestartStartupTimer = undefined
1264
+ ownerRestartChild = undefined
1265
+ return true
1266
+ }
1267
+
1268
+ /**
1269
+ * Records an owner restart failure without exposing the private command or environment.
1270
+ * @param {{command: {cwd?: string, logPath?: string, pidPath?: string}} | undefined} recovery - Accepted recovery definition.
1271
+ * @param {{code: string, exitCode?: number | null, signal?: ChildExitSignal}} failure - Secret-safe startup failure.
1272
+ */
1273
+ function reportOwnerRestartFailure(recovery, failure) {
1274
+ const diagnostic = `${JSON.stringify({at: new Date().toISOString(), ...failure, message: "guardian failed to restart daemon"})}\n`
1275
+
1276
+ if (recovery?.command.logPath) {
1277
+ try {
1278
+ fsSync.appendFileSync(recovery.command.logPath, diagnostic)
1279
+ return
1280
+ } catch (logError) {
1281
+ process.stderr.write(`${JSON.stringify({at: new Date().toISOString(), code: errorCode(logError instanceof Error ? logError : String(logError)), message: "guardian failed to write daemon restart diagnostic"})}\n`)
1282
+ return
1283
+ }
1284
+ }
1285
+ process.stderr.write(diagnostic)
1286
+ }
1287
+
1288
+ /**
1289
+ * Terminates a detached recovery candidate and all descendants in its process group.
1290
+ * @param {import("node:child_process").ChildProcess} child - Exact detached process-group leader.
1291
+ */
1292
+ function killDetachedProcessGroup(child) {
1293
+ if (!child.pid) return
1294
+ try {
1295
+ process.kill(-child.pid, "SIGKILL")
1296
+ } catch (error) {
1297
+ const failure = error instanceof Error ? error : String(error)
1298
+
1299
+ if (errorCode(failure) === "ESRCH") return
1300
+ child.kill("SIGKILL")
1301
+ process.stderr.write(`${JSON.stringify({at: new Date().toISOString(), code: errorCode(failure), message: "guardian failed to kill daemon restart process group"})}\n`)
1302
+ }
1303
+ }
1304
+
1305
+ /**
1306
+ * @param {Error | string} error - Spawn failure.
1307
+ * @returns {{code: string}} Secret-safe failure.
1308
+ */
1309
+ function restartFailure(error) {
1310
+ return {code: errorCode(error)}
1311
+ }
1312
+
1313
+ /**
1314
+ * @param {Error | string} error - Error-like value.
1315
+ * @returns {string} Stable non-secret error code.
1316
+ */
1317
+ function errorCode(error) {
1318
+ if (error && typeof error === "object" && "code" in error && (typeof error.code === "string" || typeof error.code === "number")) return String(error.code)
1319
+ return "UNKNOWN"
1320
+ }
1321
+
1322
+ /**
1323
+ * @returns {{command: {args: string[], cwd: string, env: Record<string, string>, executable: string, logPath?: string, pidPath?: string}, reconnectGraceMs: number, startupTimeoutMs: number} | undefined} Recovery definition.
1324
+ */
1325
+ function ownerRecoveryDefinition() {
1326
+ if (!ownerState || typeof ownerState !== "object" || Array.isArray(ownerState) || !("recovery" in ownerState)) return
1327
+ const recovery = ownerState.recovery
1328
+
1329
+ if (!recovery || typeof recovery !== "object" || Array.isArray(recovery) || !("command" in recovery) || !("reconnectGraceMs" in recovery)) {
1330
+ throw new Error("Guardian owner recovery definition is incomplete")
1331
+ }
1332
+ return /** @type {{command: {args: string[], cwd: string, env: Record<string, string>, executable: string, logPath?: string, pidPath?: string}, reconnectGraceMs: number, startupTimeoutMs: number}} */ (recovery)
1333
+ }
1334
+
644
1335
  /**
645
1336
  * @param {Error | string} error - Error-like value.
646
1337
  * @returns {string} Error message.