rollbridge 0.1.39 → 0.1.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -4
- package/changelog.d/20260830-guardian-retired-replacement-process-key.md +12 -1
- package/changelog.d/20260830-release-generation-activation-lifecycle.md +16 -2
- package/changelog.d/20260830055159-guardian-daemon-restart.md +2 -0
- package/docs/cli.md +23 -9
- package/docs/config.md +37 -13
- package/docs/logging.md +4 -3
- package/docs/troubleshooting.md +7 -5
- package/package.json +1 -1
- package/src/cli.js +93 -24
- package/src/config.js +14 -6
- package/src/daemon.js +643 -150
- package/src/guardian-client.js +68 -16
- package/src/managed-process.js +55 -8
- package/src/process-guardian.js +731 -43
- package/src/release-group.js +23 -7
- package/test/config-validation.test.js +4 -0
- package/test/fixtures/guardian-recovery-owner.js +86 -0
- package/test/fixtures/pre-split3-process-guardian.js +14 -0
- package/test/guardian-client.test.js +1339 -62
- package/test/managed-process.test.js +136 -7
- package/test/owner-recovery.test.js +564 -55
- package/test/owner-replacement.test.js +526 -23
- package/test/release-group.test.js +19 -2
- package/test/release-runtime-retention.test.js +121 -4
- package/test/rollbridge.test.js +21 -0
- package/test/support/process.js +41 -0
package/src/guardian-client.js
CHANGED
|
@@ -32,7 +32,7 @@ export default class GuardianClient {
|
|
|
32
32
|
* @param {{legacyGuardian?: {pid?: number, socketPath: string, token: string}, ownerState?: import("./json.js").JsonValue}} [options] - Optional authenticated legacy backend migration.
|
|
33
33
|
*/
|
|
34
34
|
async launch(options = {}) {
|
|
35
|
-
const child = spawn(process.execPath, [guardianPath, this.socketPath], {detached: true, stdio: ["ignore", "
|
|
35
|
+
const child = spawn(process.execPath, [guardianPath, this.socketPath], {detached: true, stdio: ["ignore", "inherit", "inherit", "ipc"]})
|
|
36
36
|
|
|
37
37
|
this.pid = child.pid
|
|
38
38
|
this.guardianExitPromise = new Promise((resolve) => child.once("exit", () => resolve(undefined)))
|
|
@@ -176,7 +176,17 @@ export default class GuardianClient {
|
|
|
176
176
|
* @param {import("./json.js").JsonValue} authority - Exact owner authority.
|
|
177
177
|
*/
|
|
178
178
|
async claimOwner(graceMs, authority) {
|
|
179
|
-
await this.request({authority, command: "claim-owner", graceMs})
|
|
179
|
+
await this.request({authority, command: "claim-owner", graceMs, ownerPid: process.pid})
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/** Confirms that the claimed daemon has completed startup and published its listeners. */
|
|
183
|
+
async ownerReady() {
|
|
184
|
+
await this.request({command: "owner-ready", ownerPid: process.pid})
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/** @returns {Promise<{daemonRecovery: number}>} Guardian protocol capabilities. */
|
|
188
|
+
async capabilities() {
|
|
189
|
+
return /** @type {{daemonRecovery: number}} */ (await this.request({command: "capabilities"}))
|
|
180
190
|
}
|
|
181
191
|
|
|
182
192
|
/** Starts graceful process retirement and relinquishes committed owner authority. */
|
|
@@ -221,7 +231,7 @@ export default class GuardianClient {
|
|
|
221
231
|
* @returns {Promise<{ownerState: import("./json.js").JsonValue, replacementId: string}>} Prepared transaction.
|
|
222
232
|
*/
|
|
223
233
|
async prepareOwnerReplacement(authority, nextAuthority) {
|
|
224
|
-
return /** @type {{ownerState: import("./json.js").JsonValue, replacementId: string}} */ (await this.request({authority, command: "prepare-owner-replacement", nextAuthority}))
|
|
234
|
+
return /** @type {{ownerState: import("./json.js").JsonValue, replacementId: string}} */ (await this.request({authority, command: "prepare-owner-replacement", nextAuthority, ownerPid: process.pid}))
|
|
225
235
|
}
|
|
226
236
|
|
|
227
237
|
/**
|
|
@@ -251,13 +261,24 @@ export default class GuardianClient {
|
|
|
251
261
|
await this.request({command: "commit-retired-owner-replacement", key, replacementId})
|
|
252
262
|
}
|
|
253
263
|
|
|
264
|
+
/**
|
|
265
|
+
* Yields a control-less incumbent's listeners while it still owns authority.
|
|
266
|
+
* @param {string} replacementId - Same-authority staged transaction.
|
|
267
|
+
* @param {string} key - Exact recovered guardian process proving candidate reconstruction.
|
|
268
|
+
*/
|
|
269
|
+
async prepareRetiredOwnerListenerHandoff(replacementId, key) {
|
|
270
|
+
await this.request({command: "prepare-retired-owner-listener-handoff", key, replacementId})
|
|
271
|
+
}
|
|
272
|
+
|
|
254
273
|
/**
|
|
255
274
|
* Begins acquiring the authenticated legacy backend owner channel at the disruptive boundary.
|
|
256
275
|
* @param {string} replacementId - Exact prepared candidate transaction.
|
|
257
276
|
* @param {number} graceMs - Event-driven incumbent disconnect grace.
|
|
277
|
+
* @param {string} statePath - Durable public recovery state path.
|
|
278
|
+
* @param {import("./json.js").JsonValue} recoverySnapshot - Discoverable coordinator state.
|
|
258
279
|
*/
|
|
259
|
-
async beginLegacyOwnerClaim(replacementId, graceMs) {
|
|
260
|
-
await this.request({command: "begin-legacy-owner-claim", graceMs, replacementId})
|
|
280
|
+
async beginLegacyOwnerClaim(replacementId, graceMs, statePath, recoverySnapshot) {
|
|
281
|
+
await this.request({command: "begin-legacy-owner-claim", graceMs, recoverySnapshot, replacementId, statePath})
|
|
261
282
|
}
|
|
262
283
|
|
|
263
284
|
/**
|
|
@@ -273,6 +294,22 @@ export default class GuardianClient {
|
|
|
273
294
|
await this.request({command: "finalize-owner-replacement", replacementId})
|
|
274
295
|
}
|
|
275
296
|
|
|
297
|
+
/** @param {string} replacementId - Committed transaction with complete listener state. */
|
|
298
|
+
async completeOwnerListenerRetirement(replacementId) {
|
|
299
|
+
await this.request({command: "complete-owner-listener-retirement", replacementId})
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
/**
|
|
303
|
+
* @param {string} replacementId - Committed control-less replacement transaction.
|
|
304
|
+
* @param {string} sourceId - Stable listener source identity.
|
|
305
|
+
* @param {string} releaseId - Retained release identity.
|
|
306
|
+
* @param {{http: number, websocket: number}} connections - Exact incumbent listener counts.
|
|
307
|
+
* @param {boolean} [localSource] - Whether the sender physically owns this source.
|
|
308
|
+
*/
|
|
309
|
+
async publishOwnerConnectionState(replacementId, sourceId, releaseId, connections, localSource = false) {
|
|
310
|
+
await this.request({command: "publish-owner-connection-state", connections, localSource, releaseId, replacementId, sourceId})
|
|
311
|
+
}
|
|
312
|
+
|
|
276
313
|
/** @param {string} replacementId - Prepared transaction to validate for listener yield. */
|
|
277
314
|
async validateOwnerReplacement(replacementId) {
|
|
278
315
|
await this.request({command: "validate-owner-replacement", replacementId})
|
|
@@ -294,9 +331,9 @@ export default class GuardianClient {
|
|
|
294
331
|
await this.request({command: "end-owner-mutation", mutationId})
|
|
295
332
|
}
|
|
296
333
|
|
|
297
|
-
/** @returns {Promise<{committedReplacementId: string | null, ownerClaimed: boolean}>} Transaction status. */
|
|
334
|
+
/** @returns {Promise<{committedReplacementId: string | null, ownerClaimed: boolean, retirementFailed?: boolean, retirementPending?: boolean, retirementReady?: boolean}>} Transaction status. */
|
|
298
335
|
async replacementStatus() {
|
|
299
|
-
return /** @type {{committedReplacementId: string | null, ownerClaimed: boolean}} */ (await this.request({command: "replacement-status"}))
|
|
336
|
+
return /** @type {{committedReplacementId: string | null, ownerClaimed: boolean, retirementFailed?: boolean, retirementPending?: boolean, retirementReady?: boolean}} */ (await this.request({command: "replacement-status"}))
|
|
300
337
|
}
|
|
301
338
|
|
|
302
339
|
/** @returns {Promise<import("./json.js").JsonValue>} Current private transfer state. */
|
|
@@ -444,17 +481,29 @@ class GuardianProcess extends ManagedProcess {
|
|
|
444
481
|
this.cachedStatus = asProcessStatus(await this.client.request({command: "start", key: this.key, lifecycleRole, reason}))
|
|
445
482
|
}
|
|
446
483
|
|
|
447
|
-
/**
|
|
448
|
-
|
|
449
|
-
|
|
484
|
+
/**
|
|
485
|
+
* @param {import("./managed-process.js").ManagedProcessDefinition} definition - Updated definition.
|
|
486
|
+
* @param {import("./json.js").JsonValue} [ownerState] - Private owner state to commit with the definition.
|
|
487
|
+
* @returns {Promise<void>} Resolves once the guardian commits the replacement definition.
|
|
488
|
+
*/
|
|
489
|
+
updateDefinition(definition, ownerState) {
|
|
450
490
|
const registration = this.ensureRegistered()
|
|
491
|
+
const previousUpdate = this.pendingUpdate
|
|
492
|
+
const nextDefinition = serializableDefinition({...definition, id: this.id})
|
|
493
|
+
const nextProvenance = crypto.createHash("sha256").update(JSON.stringify(nextDefinition)).digest("hex")
|
|
451
494
|
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
this.pendingUpdate = registration.then(async () => {
|
|
456
|
-
this.cachedStatus = asProcessStatus(await this.client.request({command: "update", definition: this.definition, key: this.key, previousProvenance, provenance: this.provenance}))
|
|
495
|
+
const previousUpdateSettled = previousUpdate.catch(() => {
|
|
496
|
+
// The prior caller received the update failure and local provenance stayed unchanged,
|
|
497
|
+
// so a later explicit update may retry from the last guardian-committed definition.
|
|
457
498
|
})
|
|
499
|
+
|
|
500
|
+
this.pendingUpdate = Promise.all([registration, previousUpdateSettled]).then(async () => {
|
|
501
|
+
this.cachedStatus = asProcessStatus(await this.client.request({command: "update", definition: nextDefinition, key: this.key, ownerState, previousProvenance: this.provenance, provenance: nextProvenance}))
|
|
502
|
+
super.updateDefinition(definition)
|
|
503
|
+
this.definition = nextDefinition
|
|
504
|
+
this.provenance = nextProvenance
|
|
505
|
+
})
|
|
506
|
+
return this.pendingUpdate
|
|
458
507
|
}
|
|
459
508
|
|
|
460
509
|
async quiesce() {
|
|
@@ -502,7 +551,10 @@ class GuardianProcess extends ManagedProcess {
|
|
|
502
551
|
onGuardianEvent(event) {
|
|
503
552
|
if (event.status) this.cachedStatus = asProcessStatus(event.status)
|
|
504
553
|
if (event.event === "process-log") {
|
|
505
|
-
|
|
554
|
+
const entry = asProcessLog(event.entry)
|
|
555
|
+
|
|
556
|
+
this.cachedStatus = {...this.cachedStatus, logs: [...this.cachedStatus.logs, entry].slice(-this.outputLines)}
|
|
557
|
+
this.emit("log", entry)
|
|
506
558
|
return
|
|
507
559
|
}
|
|
508
560
|
if (event.message === "process started") this.emit("started")
|
package/src/managed-process.js
CHANGED
|
@@ -5,6 +5,7 @@ import {spawn} from "node:child_process"
|
|
|
5
5
|
import {processGroupHasLiveMembers, processGroupMembers} from "./process-memory.js"
|
|
6
6
|
|
|
7
7
|
const ACTIVATION_HOOK_TIMEOUT_MS = 30000
|
|
8
|
+
const MAX_BUFFERED_OUTPUT_CHARACTERS = 64 * 1024
|
|
8
9
|
|
|
9
10
|
/**
|
|
10
11
|
* @typedef {import("./json.js").JsonValue} JsonValue
|
|
@@ -54,6 +55,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
54
55
|
this.state = /** @type {ManagedProcessState} */ ("stopped")
|
|
55
56
|
this.lastStartReason = /** @type {ManagedProcessStartReason | undefined} */ (undefined)
|
|
56
57
|
this.logs = /** @type {ManagedProcessLog[]} */ ([])
|
|
58
|
+
this.outputBuffers = {stderr: "", stdout: ""}
|
|
57
59
|
this.restarts = 0
|
|
58
60
|
this.recentRestarts = /** @type {number[]} */ ([])
|
|
59
61
|
this.rssBytes = /** @type {number | undefined} */ (undefined)
|
|
@@ -67,9 +69,11 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
67
69
|
this.intentionalStop = false
|
|
68
70
|
this.lifecycleRole = /** @type {LifecycleRole} */ ("candidate")
|
|
69
71
|
this.intentionalStopSignal = /** @type {ProcessExitSignal | undefined} */ (undefined)
|
|
72
|
+
this.lifecycleRestoreBarrier = /** @type {Promise<void> | undefined} */ (undefined)
|
|
70
73
|
this.quiescePromise = /** @type {Promise<void> | undefined} */ (undefined)
|
|
71
74
|
this.quiesceError = /** @type {Error | undefined} */ (undefined)
|
|
72
75
|
this.stopPromise = /** @type {Promise<void> | undefined} */ (undefined)
|
|
76
|
+
this.operationRevision = 0
|
|
73
77
|
this.restartTimer = undefined
|
|
74
78
|
this.child = undefined
|
|
75
79
|
this.exitPromise = undefined
|
|
@@ -84,6 +88,10 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
84
88
|
* @returns {Promise<void>} Resolves after spawn and lifecycle-role restoration.
|
|
85
89
|
*/
|
|
86
90
|
async start(reason = "deploy", lifecycleRole) {
|
|
91
|
+
const operationRevision = ++this.operationRevision
|
|
92
|
+
|
|
93
|
+
if (this.stopPromise) await this.stopPromise
|
|
94
|
+
if (operationRevision !== this.operationRevision) return
|
|
87
95
|
if (lifecycleRole) this.lifecycleRole = lifecycleRole
|
|
88
96
|
if (this.child) return
|
|
89
97
|
|
|
@@ -97,6 +105,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
97
105
|
this.state = "starting"
|
|
98
106
|
|
|
99
107
|
await new Promise((resolve, reject) => {
|
|
108
|
+
const outputBuffers = {stderr: "", stdout: ""}
|
|
100
109
|
const child = spawn(this.command, {
|
|
101
110
|
cwd: this.cwd,
|
|
102
111
|
detached: true,
|
|
@@ -118,18 +127,28 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
118
127
|
void (async () => {
|
|
119
128
|
this.startedAtMs = Date.now()
|
|
120
129
|
this.lastStartReason = reason
|
|
130
|
+
const lifecycleRestore = this.restoreLifecycleRole()
|
|
131
|
+
const lifecycleRestoreBarrier = lifecycleRestore.then(() => undefined, () => undefined)
|
|
132
|
+
|
|
133
|
+
this.lifecycleRestoreBarrier = lifecycleRestoreBarrier
|
|
121
134
|
try {
|
|
122
|
-
await
|
|
135
|
+
await lifecycleRestore
|
|
123
136
|
} catch (error) {
|
|
124
137
|
this.state = "failed"
|
|
125
138
|
this.logger("process lifecycle role restoration failed", {error: error instanceof Error ? error.message : String(error), id: this.id, role: this.lifecycleRole})
|
|
126
139
|
reject(error)
|
|
127
140
|
return
|
|
141
|
+
} finally {
|
|
142
|
+
if (this.lifecycleRestoreBarrier === lifecycleRestoreBarrier) this.lifecycleRestoreBarrier = undefined
|
|
128
143
|
}
|
|
129
144
|
if (this.child !== child) {
|
|
130
145
|
reject(new Error(`Process ${this.id} exited before lifecycle role ${this.lifecycleRole} was restored`))
|
|
131
146
|
return
|
|
132
147
|
}
|
|
148
|
+
if (this.intentionalStop || this.state !== "starting") {
|
|
149
|
+
reject(new Error(`Process ${this.id} was quiesced before lifecycle role ${this.lifecycleRole} was restored`))
|
|
150
|
+
return
|
|
151
|
+
}
|
|
133
152
|
this.state = "running"
|
|
134
153
|
this.logger("process started", {command: this.command, id: this.id, pid: child.pid || null, reason})
|
|
135
154
|
this.startMemoryMonitor()
|
|
@@ -148,17 +167,20 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
148
167
|
})
|
|
149
168
|
child.stdout.setEncoding("utf8")
|
|
150
169
|
child.stderr.setEncoding("utf8")
|
|
151
|
-
child.stdout.on("data", (chunk) => this.appendLog("stdout", chunk))
|
|
152
|
-
child.
|
|
170
|
+
child.stdout.on("data", (chunk) => this.appendLog("stdout", chunk, outputBuffers))
|
|
171
|
+
child.stdout.on("end", () => this.flushLogBuffer("stdout", outputBuffers))
|
|
172
|
+
child.stderr.on("data", (chunk) => this.appendLog("stderr", chunk, outputBuffers))
|
|
173
|
+
child.stderr.on("end", () => this.flushLogBuffer("stderr", outputBuffers))
|
|
153
174
|
})
|
|
154
175
|
}
|
|
155
176
|
|
|
156
177
|
/**
|
|
157
178
|
* Updates the command template used for future restarts without touching the currently running child.
|
|
158
179
|
* @param {ManagedProcessDefinition} definition - Replacement process definition.
|
|
159
|
-
* @
|
|
180
|
+
* @param {import("./json.js").JsonValue} [_ownerState] - Private owner state committed atomically by remote implementations.
|
|
181
|
+
* @returns {void | Promise<void>} Definition replacement completion for remote implementations.
|
|
160
182
|
*/
|
|
161
|
-
updateDefinition(definition) {
|
|
183
|
+
updateDefinition(definition, _ownerState) {
|
|
162
184
|
this.command = definition.command
|
|
163
185
|
this.cwd = definition.cwd
|
|
164
186
|
this.env = definition.env
|
|
@@ -176,10 +198,19 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
176
198
|
/**
|
|
177
199
|
* @param {"stdout" | "stderr"} stream - Stream name.
|
|
178
200
|
* @param {string} chunk - Output chunk.
|
|
201
|
+
* @param {{stderr: string, stdout: string}} [buffers] - Per-process stream fragments.
|
|
179
202
|
* @returns {void}
|
|
180
203
|
*/
|
|
181
|
-
appendLog(stream, chunk) {
|
|
182
|
-
|
|
204
|
+
appendLog(stream, chunk, buffers = this.outputBuffers) {
|
|
205
|
+
const lines = `${buffers[stream]}${String(chunk)}`.split(/\r?\n/)
|
|
206
|
+
let fragment = lines.pop() ?? ""
|
|
207
|
+
|
|
208
|
+
while (fragment.length > MAX_BUFFERED_OUTPUT_CHARACTERS) {
|
|
209
|
+
lines.push(fragment.slice(0, MAX_BUFFERED_OUTPUT_CHARACTERS))
|
|
210
|
+
fragment = fragment.slice(MAX_BUFFERED_OUTPUT_CHARACTERS)
|
|
211
|
+
}
|
|
212
|
+
buffers[stream] = fragment
|
|
213
|
+
for (const line of lines) {
|
|
183
214
|
if (!line) continue
|
|
184
215
|
|
|
185
216
|
const entry = {at: new Date().toISOString(), line, stream}
|
|
@@ -193,6 +224,15 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
193
224
|
}
|
|
194
225
|
}
|
|
195
226
|
|
|
227
|
+
/**
|
|
228
|
+
* Retains a final output line that did not end with a newline.
|
|
229
|
+
* @param {"stdout" | "stderr"} stream - Stream name.
|
|
230
|
+
* @param {{stderr: string, stdout: string}} buffers - Per-process stream fragments.
|
|
231
|
+
*/
|
|
232
|
+
flushLogBuffer(stream, buffers) {
|
|
233
|
+
if (buffers[stream]) this.appendLog(stream, "\n", buffers)
|
|
234
|
+
}
|
|
235
|
+
|
|
196
236
|
/**
|
|
197
237
|
* @param {number | null} code - Exit code.
|
|
198
238
|
* @param {ProcessExitSignal} signal - Exit signal.
|
|
@@ -375,6 +415,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
375
415
|
* @returns {Promise<void>} Resolves when stopped.
|
|
376
416
|
*/
|
|
377
417
|
async stop(options = {}) {
|
|
418
|
+
this.operationRevision += 1
|
|
378
419
|
if (!this.stopPromise) this.stopPromise = this.performStop(options)
|
|
379
420
|
return await this.stopPromise
|
|
380
421
|
}
|
|
@@ -442,6 +483,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
442
483
|
clearTimeout(this.restartTimer)
|
|
443
484
|
this.restartTimer = undefined
|
|
444
485
|
}
|
|
486
|
+
if (this.lifecycleRestoreBarrier) await this.lifecycleRestoreBarrier
|
|
445
487
|
if (!this.child?.pid) {
|
|
446
488
|
this.state = "stopped"
|
|
447
489
|
if (this.lifecycle.activateCommand) this.lifecycleRole = "retired"
|
|
@@ -475,9 +517,14 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
475
517
|
const command = this.lifecycle.activateCommand
|
|
476
518
|
|
|
477
519
|
if (!command) return
|
|
478
|
-
const
|
|
520
|
+
const child = this.child
|
|
521
|
+
const pid = this.pid
|
|
522
|
+
|
|
523
|
+
if (!child?.pid || child.pid !== pid || this.state !== "running") throw new Error(`Process ${this.id} is not running for activation`)
|
|
524
|
+
const error = await this.runHook(command, ACTIVATION_HOOK_TIMEOUT_MS, "activate command", pid)
|
|
479
525
|
|
|
480
526
|
if (error) throw error
|
|
527
|
+
if (this.child !== child || this.pid !== pid || this.state !== "running") throw new Error(`Process ${this.id} exited before activation completed`)
|
|
481
528
|
this.lifecycleRole = "active"
|
|
482
529
|
}
|
|
483
530
|
|