rollbridge 0.1.30 → 0.1.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +11 -6
- package/README.md +28 -12
- package/changelog.d/20260830-release-generation-activation-lifecycle.md +9 -0
- package/docs/cli.md +10 -7
- package/docs/config.md +41 -13
- package/docs/tensorbuzz-runbook.md +4 -2
- package/docs/velocious.md +19 -7
- package/docs/workers.md +18 -9
- package/examples/tensorbuzz.com.js +9 -5
- package/package.json +1 -1
- package/src/config.js +32 -2
- package/src/daemon.js +305 -42
- package/src/guardian-client.js +46 -3
- package/src/managed-process.js +85 -13
- package/src/process-guardian.js +45 -1
- package/src/release-group.js +53 -14
- package/test/config-validation.test.js +44 -0
- package/test/guardian-client.test.js +70 -0
- package/test/managed-process.test.js +33 -0
- package/test/owner-recovery.test.js +396 -6
- package/test/owner-replacement.test.js +153 -3
- package/test/rollbridge.test.js +333 -6
package/src/guardian-client.js
CHANGED
|
@@ -194,6 +194,11 @@ export default class GuardianClient {
|
|
|
194
194
|
await this.request({command: "commit-owner-replacement", replacementId})
|
|
195
195
|
}
|
|
196
196
|
|
|
197
|
+
/** @param {string} replacementId - Same-authority transaction whose incumbent listener is absent. */
|
|
198
|
+
async commitRetiredOwnerReplacement(replacementId) {
|
|
199
|
+
await this.request({command: "commit-retired-owner-replacement", replacementId})
|
|
200
|
+
}
|
|
201
|
+
|
|
197
202
|
/** @param {string} replacementId - Committed transaction awaiting incumbent retirement. */
|
|
198
203
|
async finalizeOwnerReplacement(replacementId) {
|
|
199
204
|
await this.request({command: "finalize-owner-replacement", replacementId})
|
|
@@ -305,7 +310,7 @@ export default class GuardianClient {
|
|
|
305
310
|
|
|
306
311
|
this.buffer = this.buffer.slice(newline + 1)
|
|
307
312
|
if (message.event) {
|
|
308
|
-
if (message.event === "process" || message.event === "status") this.processes.get(message.key)?.onGuardianEvent(message)
|
|
313
|
+
if (message.event === "process" || message.event === "process-log" || message.event === "status") this.processes.get(message.key)?.onGuardianEvent(message)
|
|
309
314
|
for (const handler of this.eventHandlers.get(message.event) || []) handler(message)
|
|
310
315
|
const waiter = this.events.get(message.event)?.shift()
|
|
311
316
|
|
|
@@ -349,10 +354,15 @@ class GuardianProcess extends ManagedProcess {
|
|
|
349
354
|
await this.ensureRegistered()
|
|
350
355
|
}
|
|
351
356
|
|
|
352
|
-
|
|
357
|
+
/**
|
|
358
|
+
* @param {import("./managed-process.js").ManagedProcessStartReason} [reason] - Start reason.
|
|
359
|
+
* @param {import("./managed-process.js").LifecycleRole} [lifecycleRole] - Desired role restored before running.
|
|
360
|
+
*/
|
|
361
|
+
async start(reason = "deploy", lifecycleRole) {
|
|
353
362
|
await this.ensureRegistered()
|
|
354
363
|
await this.pendingUpdate
|
|
355
|
-
|
|
364
|
+
if (lifecycleRole) this.lifecycleRole = lifecycleRole
|
|
365
|
+
this.cachedStatus = asProcessStatus(await this.client.request({command: "start", key: this.key, lifecycleRole, reason}))
|
|
356
366
|
}
|
|
357
367
|
|
|
358
368
|
/** @param {import("./managed-process.js").ManagedProcessDefinition} definition - Updated definition. */
|
|
@@ -378,6 +388,27 @@ class GuardianProcess extends ManagedProcess {
|
|
|
378
388
|
await this.quiesce()
|
|
379
389
|
}
|
|
380
390
|
|
|
391
|
+
async requiesceStrict() {
|
|
392
|
+
await this.ensureRegistered()
|
|
393
|
+
await this.pendingUpdate
|
|
394
|
+
this.cachedStatus = asProcessStatus(await this.client.request({command: "requiesce", key: this.key}))
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
async activateStrict() {
|
|
398
|
+
await this.ensureRegistered()
|
|
399
|
+
await this.pendingUpdate
|
|
400
|
+
this.cachedStatus = asProcessStatus(await this.client.request({command: "activate", key: this.key}))
|
|
401
|
+
this.lifecycleRole = "active"
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
/** @param {import("./managed-process.js").LifecycleRole} role - Exact generation role. */
|
|
405
|
+
async setLifecycleRole(role) {
|
|
406
|
+
await this.ensureRegistered()
|
|
407
|
+
await this.pendingUpdate
|
|
408
|
+
this.cachedStatus = asProcessStatus(await this.client.request({command: "set-lifecycle-role", key: this.key, lifecycleRole: role}))
|
|
409
|
+
this.lifecycleRole = role
|
|
410
|
+
}
|
|
411
|
+
|
|
381
412
|
async stop(options = {}) {
|
|
382
413
|
await this.ensureRegistered()
|
|
383
414
|
await this.pendingUpdate
|
|
@@ -391,6 +422,10 @@ class GuardianProcess extends ManagedProcess {
|
|
|
391
422
|
/** @param {Record<string, import("./json.js").JsonValue>} event - Guardian event. */
|
|
392
423
|
onGuardianEvent(event) {
|
|
393
424
|
if (event.status) this.cachedStatus = asProcessStatus(event.status)
|
|
425
|
+
if (event.event === "process-log") {
|
|
426
|
+
this.emit("log", asProcessLog(event.entry))
|
|
427
|
+
return
|
|
428
|
+
}
|
|
394
429
|
if (event.message === "process started") this.emit("started")
|
|
395
430
|
if (event.message === "process exited") this.emit("exit", event.data)
|
|
396
431
|
this.logger(typeof event.message === "string" ? event.message : "guardian process status", event.data && typeof event.data === "object" && !Array.isArray(event.data) ? event.data : {})
|
|
@@ -425,6 +460,14 @@ function asProcessStatus(value) {
|
|
|
425
460
|
return JSON.parse(JSON.stringify(value))
|
|
426
461
|
}
|
|
427
462
|
|
|
463
|
+
/**
|
|
464
|
+
* @param {import("./json.js").JsonValue} value - Protocol value.
|
|
465
|
+
* @returns {import("./managed-process.js").ManagedProcessLog} Process output entry.
|
|
466
|
+
*/
|
|
467
|
+
function asProcessLog(value) {
|
|
468
|
+
return JSON.parse(JSON.stringify(value))
|
|
469
|
+
}
|
|
470
|
+
|
|
428
471
|
/**
|
|
429
472
|
* @param {Error | string} error - Error-like value.
|
|
430
473
|
* @returns {string} Error message.
|
package/src/managed-process.js
CHANGED
|
@@ -4,15 +4,18 @@ import {EventEmitter} from "node:events"
|
|
|
4
4
|
import {spawn} from "node:child_process"
|
|
5
5
|
import {processGroupHasLiveMembers, processGroupMembers} from "./process-memory.js"
|
|
6
6
|
|
|
7
|
+
const ACTIVATION_HOOK_TIMEOUT_MS = 30000
|
|
8
|
+
|
|
7
9
|
/**
|
|
8
10
|
* @typedef {import("./json.js").JsonValue} JsonValue
|
|
9
11
|
* @typedef {"starting" | "running" | "quiesced" | "stopping" | "stopped" | "failed"} ManagedProcessState
|
|
10
12
|
* @typedef {"deploy" | "crash" | "manual" | "memory"} ManagedProcessStartReason
|
|
13
|
+
* @typedef {"active" | "candidate" | "retired"} LifecycleRole
|
|
11
14
|
* @typedef {import("node:child_process").ChildProcess["signalCode"]} ProcessExitSignal
|
|
12
15
|
* @typedef {{at: string, line: string, stream: "stdout" | "stderr"}} ManagedProcessLog
|
|
13
16
|
* @typedef {import("./config.js").StopTimeoutMs} StopTimeoutMs
|
|
14
17
|
* @typedef {{command: string, cwd: string | undefined, env: Record<string, string | undefined>, lifecycle: import("./config.js").LifecycleConfig, logger: (message: string, data?: Record<string, import("./json.js").JsonValue>) => void, memory: import("./config.js").MemoryConfig | undefined, outputLines: number, restart: import("./config.js").RestartConfig, restartDelayMs: number, shouldRestart: () => boolean, stopSignal: string, stopTimeoutMs: StopTimeoutMs}} ManagedProcessDefinition
|
|
15
|
-
* @typedef {{children: import("./process-memory.js").ProcessGroupMember[], command: string, cwd: string | undefined, exitCode: number | null | undefined, exitSignal: ProcessExitSignal | undefined, id: string, lastMemoryRestartAt: string | undefined, lastStartReason: ManagedProcessStartReason | undefined, logs: ManagedProcessLog[], memoryRestarts: number, pid: number | undefined, restarts: number, rssBytes: number | undefined, startedAt: string | undefined, state: ManagedProcessState, uptimeMs: number | undefined}} ManagedProcessStatus
|
|
18
|
+
* @typedef {{children: import("./process-memory.js").ProcessGroupMember[], command: string, cwd: string | undefined, exitCode: number | null | undefined, exitSignal: ProcessExitSignal | undefined, id: string, lastMemoryRestartAt: string | undefined, lastStartReason: ManagedProcessStartReason | undefined, lifecycleRole?: LifecycleRole, logs: ManagedProcessLog[], memoryRestarts: number, pid: number | undefined, restarts: number, rssBytes: number | undefined, startedAt: string | undefined, state: ManagedProcessState, uptimeMs: number | undefined}} ManagedProcessStatus
|
|
16
19
|
*/
|
|
17
20
|
|
|
18
21
|
export default class ManagedProcess extends EventEmitter {
|
|
@@ -62,6 +65,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
62
65
|
this.memoryWarned = false
|
|
63
66
|
this.startedAtMs = /** @type {number | undefined} */ (undefined)
|
|
64
67
|
this.intentionalStop = false
|
|
68
|
+
this.lifecycleRole = /** @type {LifecycleRole} */ ("candidate")
|
|
65
69
|
this.intentionalStopSignal = /** @type {ProcessExitSignal | undefined} */ (undefined)
|
|
66
70
|
this.quiescePromise = /** @type {Promise<void> | undefined} */ (undefined)
|
|
67
71
|
this.quiesceError = /** @type {Error | undefined} */ (undefined)
|
|
@@ -76,9 +80,11 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
76
80
|
|
|
77
81
|
/**
|
|
78
82
|
* @param {ManagedProcessStartReason} [reason] - Why the process is being started (deploy by default; "crash" on auto-restart, "manual" via the restart command).
|
|
79
|
-
* @
|
|
83
|
+
* @param {LifecycleRole} [lifecycleRole] - Exact desired role to restore before reporting the process running.
|
|
84
|
+
* @returns {Promise<void>} Resolves after spawn and lifecycle-role restoration.
|
|
80
85
|
*/
|
|
81
|
-
async start(reason = "deploy") {
|
|
86
|
+
async start(reason = "deploy", lifecycleRole) {
|
|
87
|
+
if (lifecycleRole) this.lifecycleRole = lifecycleRole
|
|
82
88
|
if (this.child) return
|
|
83
89
|
|
|
84
90
|
this.intentionalStop = false
|
|
@@ -109,16 +115,35 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
109
115
|
})
|
|
110
116
|
|
|
111
117
|
child.once("spawn", () => {
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
118
|
+
void (async () => {
|
|
119
|
+
this.startedAtMs = Date.now()
|
|
120
|
+
this.lastStartReason = reason
|
|
121
|
+
try {
|
|
122
|
+
await this.restoreLifecycleRole()
|
|
123
|
+
} catch (error) {
|
|
124
|
+
this.state = "failed"
|
|
125
|
+
this.logger("process lifecycle role restoration failed", {error: error instanceof Error ? error.message : String(error), id: this.id, role: this.lifecycleRole})
|
|
126
|
+
reject(error)
|
|
127
|
+
return
|
|
128
|
+
}
|
|
129
|
+
if (this.child !== child) {
|
|
130
|
+
reject(new Error(`Process ${this.id} exited before lifecycle role ${this.lifecycleRole} was restored`))
|
|
131
|
+
return
|
|
132
|
+
}
|
|
133
|
+
this.state = "running"
|
|
134
|
+
this.logger("process started", {command: this.command, id: this.id, pid: child.pid || null, reason})
|
|
135
|
+
this.startMemoryMonitor()
|
|
136
|
+
this.emit("started")
|
|
137
|
+
resolve(undefined)
|
|
138
|
+
})()
|
|
119
139
|
})
|
|
120
140
|
child.once("error", (error) => {
|
|
121
141
|
this.state = "failed"
|
|
142
|
+
if (this.child === child) {
|
|
143
|
+
this.child = undefined
|
|
144
|
+
this.pid = undefined
|
|
145
|
+
this.exitPromise = undefined
|
|
146
|
+
}
|
|
122
147
|
reject(error)
|
|
123
148
|
})
|
|
124
149
|
child.stdout.setEncoding("utf8")
|
|
@@ -157,11 +182,14 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
157
182
|
for (const line of String(chunk).split(/\r?\n/)) {
|
|
158
183
|
if (!line) continue
|
|
159
184
|
|
|
160
|
-
|
|
185
|
+
const entry = {at: new Date().toISOString(), line, stream}
|
|
186
|
+
|
|
187
|
+
this.logs.push(entry)
|
|
161
188
|
|
|
162
189
|
if (this.logs.length > this.outputLines) {
|
|
163
190
|
this.logs.splice(0, this.logs.length - this.outputLines)
|
|
164
191
|
}
|
|
192
|
+
this.emit("log", entry)
|
|
165
193
|
}
|
|
166
194
|
}
|
|
167
195
|
|
|
@@ -321,6 +349,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
321
349
|
if (this.memoryRestarting) return
|
|
322
350
|
|
|
323
351
|
this.memoryRestarting = true
|
|
352
|
+
const lifecycleRole = this.lifecycleRole
|
|
324
353
|
|
|
325
354
|
try {
|
|
326
355
|
await this.stop()
|
|
@@ -333,7 +362,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
333
362
|
this.memoryRestarts += 1
|
|
334
363
|
this.lastMemoryRestartAtMs = Date.now()
|
|
335
364
|
this.memoryWarned = false
|
|
336
|
-
await this.start("memory")
|
|
365
|
+
await this.start("memory", lifecycleRole)
|
|
337
366
|
} catch (error) {
|
|
338
367
|
this.logger("memory restart failed", {error: error instanceof Error ? error.message : String(error), id: this.id})
|
|
339
368
|
} finally {
|
|
@@ -415,11 +444,15 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
415
444
|
}
|
|
416
445
|
if (!this.child?.pid) {
|
|
417
446
|
this.state = "stopped"
|
|
447
|
+
if (this.lifecycle.activateCommand) this.lifecycleRole = "retired"
|
|
418
448
|
return
|
|
419
449
|
}
|
|
420
450
|
this.state = "stopping"
|
|
421
451
|
if (this.lifecycle.quietCommand) this.quiesceError = await this.runHook(this.lifecycle.quietCommand, this.hookTimeoutMs(), "quiet command")
|
|
422
|
-
if (!this.quiesceError)
|
|
452
|
+
if (!this.quiesceError) {
|
|
453
|
+
this.state = "quiesced"
|
|
454
|
+
if (this.lifecycle.activateCommand) this.lifecycleRole = "retired"
|
|
455
|
+
}
|
|
423
456
|
})()
|
|
424
457
|
return await this.quiescePromise
|
|
425
458
|
}
|
|
@@ -430,6 +463,44 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
430
463
|
if (this.quiesceError) throw this.quiesceError
|
|
431
464
|
}
|
|
432
465
|
|
|
466
|
+
/** Re-runs the idempotent quiet hook for an explicitly resumed durable transition. */
|
|
467
|
+
async requiesceStrict() {
|
|
468
|
+
this.quiescePromise = undefined
|
|
469
|
+
this.quiesceError = undefined
|
|
470
|
+
await this.quiesceStrict()
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
/** Runs the opt-in generation activation command and rejects on any hook failure. */
|
|
474
|
+
async activateStrict() {
|
|
475
|
+
const command = this.lifecycle.activateCommand
|
|
476
|
+
|
|
477
|
+
if (!command) return
|
|
478
|
+
const error = await this.runHook(command, ACTIVATION_HOOK_TIMEOUT_MS, "activate command", this.pid)
|
|
479
|
+
|
|
480
|
+
if (error) throw error
|
|
481
|
+
this.lifecycleRole = "active"
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
/**
|
|
485
|
+
* Records the durable desired role without firing a lifecycle command.
|
|
486
|
+
* @param {LifecycleRole} role - Exact role owned by this process generation.
|
|
487
|
+
*/
|
|
488
|
+
async setLifecycleRole(role) {
|
|
489
|
+
this.lifecycleRole = role
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
/** Restores an active or retired role after this exact process starts. */
|
|
493
|
+
async restoreLifecycleRole() {
|
|
494
|
+
if (!this.lifecycle.activateCommand || this.lifecycleRole === "candidate") return
|
|
495
|
+
const command = this.lifecycleRole === "active" ? this.lifecycle.activateCommand : this.lifecycle.quietCommand
|
|
496
|
+
|
|
497
|
+
if (!command) throw new Error(`Process ${this.id} cannot restore lifecycle role ${this.lifecycleRole} without its paired command`)
|
|
498
|
+
const timeoutMs = this.lifecycleRole === "active" ? ACTIVATION_HOOK_TIMEOUT_MS : this.hookTimeoutMs()
|
|
499
|
+
const error = await this.runHook(command, timeoutMs, `${this.lifecycleRole === "active" ? "activate" : "quiet"} command`, this.pid)
|
|
500
|
+
|
|
501
|
+
if (error) throw error
|
|
502
|
+
}
|
|
503
|
+
|
|
433
504
|
/** @returns {number} Timeout used for lifecycle hooks. */
|
|
434
505
|
hookTimeoutMs() {
|
|
435
506
|
if (this.stopTimeoutMs === "indefinite") return 30000
|
|
@@ -642,6 +713,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
642
713
|
id: this.id,
|
|
643
714
|
lastMemoryRestartAt: this.lastMemoryRestartAtMs === undefined ? undefined : new Date(this.lastMemoryRestartAtMs).toISOString(),
|
|
644
715
|
lastStartReason: this.lastStartReason,
|
|
716
|
+
...(this.lifecycle.activateCommand ? {lifecycleRole: this.lifecycleRole} : {}),
|
|
645
717
|
logs: this.logs.slice(-this.outputLines),
|
|
646
718
|
memoryRestarts: this.memoryRestarts,
|
|
647
719
|
pid: this.pid,
|
package/src/process-guardian.js
CHANGED
|
@@ -24,6 +24,7 @@ import ManagedProcess from "./managed-process.js"
|
|
|
24
24
|
* @property {import("./json.js").JsonValue} [authority] - Expected current authority.
|
|
25
25
|
* @property {import("./json.js").JsonValue} [nextAuthority] - Requested replacement authority.
|
|
26
26
|
* @property {import("./managed-process.js").ManagedProcessStartReason} [reason] - Start reason.
|
|
27
|
+
* @property {import("./managed-process.js").LifecycleRole} [lifecycleRole] - Desired generation role restored during start.
|
|
27
28
|
* @property {string} token - Authentication token.
|
|
28
29
|
*/
|
|
29
30
|
|
|
@@ -297,6 +298,23 @@ async function execute(request, socket) {
|
|
|
297
298
|
return {aborted: true}
|
|
298
299
|
}
|
|
299
300
|
|
|
301
|
+
if (request.command === "commit-retired-owner-replacement") {
|
|
302
|
+
requireReplacement(socket, request)
|
|
303
|
+
if (!replacementOwnerState) throw new Error("Retired owner replacement transaction is not staged")
|
|
304
|
+
if (!isDeepStrictEqual(ownerAuthority(ownerState), replacementAuthority)) throw new Error("Retired owner replacement requires unchanged owner authority")
|
|
305
|
+
const controlPath = ownerControlPath(ownerState)
|
|
306
|
+
|
|
307
|
+
try {
|
|
308
|
+
await fs.lstat(controlPath)
|
|
309
|
+
throw new Error(`Retired owner control socket ${controlPath} still exists`)
|
|
310
|
+
} catch (error) {
|
|
311
|
+
if (!error || typeof error !== "object" || !("code" in error) || error.code !== "ENOENT") throw error
|
|
312
|
+
}
|
|
313
|
+
commitReplacement()
|
|
314
|
+
finalizeReplacementRetirement()
|
|
315
|
+
return {committed: true}
|
|
316
|
+
}
|
|
317
|
+
|
|
300
318
|
if (request.command === "commit-owner-replacement") {
|
|
301
319
|
requireOwner(socket, request.command)
|
|
302
320
|
if (!replacementClient || request.replacementId !== replacementId || !replacementOwnerState) throw new Error("Owner replacement transaction is not the prepared ready candidate")
|
|
@@ -379,6 +397,9 @@ async function execute(request, socket) {
|
|
|
379
397
|
? legacyGuardian.process(request.key, managedDefinition)
|
|
380
398
|
: new ManagedProcess(managedDefinition)
|
|
381
399
|
|
|
400
|
+
managedProcess.on("log", (entry) => {
|
|
401
|
+
broadcast({entry, event: "process-log", key: request.key, status: managedProcess.status()})
|
|
402
|
+
})
|
|
382
403
|
if (recoversLegacyProcess && "recover" in managedProcess && typeof managedProcess.recover === "function") await managedProcess.recover()
|
|
383
404
|
|
|
384
405
|
record.process = managedProcess
|
|
@@ -396,10 +417,18 @@ async function execute(request, socket) {
|
|
|
396
417
|
|
|
397
418
|
if (request.command === "start") {
|
|
398
419
|
record.desired = true
|
|
399
|
-
await record.process.start(request.reason)
|
|
420
|
+
await record.process.start(request.reason, request.lifecycleRole)
|
|
421
|
+
} else if (request.command === "activate") {
|
|
422
|
+
await record.process.activateStrict()
|
|
400
423
|
} else if (request.command === "quiesce") {
|
|
401
424
|
record.desired = false
|
|
402
425
|
await record.process.quiesceStrict()
|
|
426
|
+
} else if (request.command === "requiesce") {
|
|
427
|
+
record.desired = false
|
|
428
|
+
await record.process.requiesceStrict()
|
|
429
|
+
} else if (request.command === "set-lifecycle-role") {
|
|
430
|
+
if (request.lifecycleRole !== "active" && request.lifecycleRole !== "candidate" && request.lifecycleRole !== "retired") throw new Error("Guardian lifecycle role is invalid")
|
|
431
|
+
await record.process.setLifecycleRole(request.lifecycleRole)
|
|
403
432
|
} else if (request.command === "stop") {
|
|
404
433
|
record.desired = false
|
|
405
434
|
await record.process.stop(request.options)
|
|
@@ -510,6 +539,21 @@ function ownerAuthority(state) {
|
|
|
510
539
|
return state.authority
|
|
511
540
|
}
|
|
512
541
|
|
|
542
|
+
/**
|
|
543
|
+
* @param {import("./json.js").JsonValue | undefined} state - Committed transferable state.
|
|
544
|
+
* @returns {string} Exact incumbent public control path.
|
|
545
|
+
*/
|
|
546
|
+
function ownerControlPath(state) {
|
|
547
|
+
if (!state || typeof state !== "object" || Array.isArray(state) || !("snapshot" in state)) throw new Error("Guardian owner state is missing its committed snapshot")
|
|
548
|
+
const snapshot = state.snapshot
|
|
549
|
+
|
|
550
|
+
if (!snapshot || typeof snapshot !== "object" || Array.isArray(snapshot) || !("control" in snapshot)) throw new Error("Guardian owner snapshot is missing its control identity")
|
|
551
|
+
const control = snapshot.control
|
|
552
|
+
|
|
553
|
+
if (!control || typeof control !== "object" || Array.isArray(control) || !("path" in control) || typeof control.path !== "string") throw new Error("Guardian owner snapshot has an invalid control identity")
|
|
554
|
+
return control.path
|
|
555
|
+
}
|
|
556
|
+
|
|
513
557
|
/**
|
|
514
558
|
* Stops accepting connections and closes every authority channel except the response caller.
|
|
515
559
|
* @param {net.Socket} caller - Shutdown requester retained until it receives the response.
|
package/src/release-group.js
CHANGED
|
@@ -116,8 +116,9 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
116
116
|
/**
|
|
117
117
|
* Reconstructs this release around processes still owned by the durable guardian.
|
|
118
118
|
* @param {ReleaseStatus} snapshot - Persisted release snapshot.
|
|
119
|
+
* @param {{synchronizeLifecycleRole?: boolean}} [options] - Guardian role synchronization options.
|
|
119
120
|
*/
|
|
120
|
-
async restore(snapshot) {
|
|
121
|
+
async restore(snapshot, {synchronizeLifecycleRole = true} = {}) {
|
|
121
122
|
if (!snapshot || snapshot.releaseId !== this.releaseId || snapshot.releasePath !== this.releasePath || snapshot.revision !== this.revision) {
|
|
122
123
|
throw new Error(`Persisted release identity mismatch for ${this.releaseId}`)
|
|
123
124
|
}
|
|
@@ -175,6 +176,17 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
175
176
|
if (processConfig.nonBlockingDrain) this.nonBlockingDrainIds.add(processStatus.id)
|
|
176
177
|
if ("recover" in processInstance && typeof processInstance.recover === "function") await processInstance.recover()
|
|
177
178
|
}
|
|
179
|
+
if (synchronizeLifecycleRole) await this.synchronizeLifecycleRoles()
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/** Synchronizes only the activation owner's durable role without firing its hook. */
|
|
183
|
+
async synchronizeLifecycleRoles() {
|
|
184
|
+
const processConfig = this.config.processes.find((candidate) => candidate.lifecycle.activateCommand !== undefined)
|
|
185
|
+
|
|
186
|
+
if (!processConfig) return
|
|
187
|
+
const lifecycleRole = this.state === "active" ? "active" : this.state === "draining" ? "retired" : "candidate"
|
|
188
|
+
|
|
189
|
+
await Promise.all(this.getProcesses(processConfig.id).map(({process}) => process.setLifecycleRole(lifecycleRole)))
|
|
178
190
|
}
|
|
179
191
|
|
|
180
192
|
/**
|
|
@@ -253,6 +265,17 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
253
265
|
this.activatedAt = new Date().toISOString()
|
|
254
266
|
}
|
|
255
267
|
|
|
268
|
+
/** Runs the configured release-generation activation acknowledgement, if any. */
|
|
269
|
+
async activateGeneration() {
|
|
270
|
+
const processConfig = this.config.processes.find((candidate) => candidate.lifecycle.activateCommand !== undefined)
|
|
271
|
+
|
|
272
|
+
if (!processConfig) return
|
|
273
|
+
const [instance] = this.getProcesses(processConfig.id)
|
|
274
|
+
|
|
275
|
+
if (!instance) throw new Error(`Generation activation process ${processConfig.id} is not running for release ${this.releaseId}`)
|
|
276
|
+
await instance.process.activateStrict()
|
|
277
|
+
}
|
|
278
|
+
|
|
256
279
|
/** @returns {Promise<void>} Allocates all configured per-process ports. */
|
|
257
280
|
async allocatePorts() {
|
|
258
281
|
if (this.portsAllocated) return
|
|
@@ -295,12 +318,12 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
295
318
|
}
|
|
296
319
|
|
|
297
320
|
/**
|
|
298
|
-
* Builds
|
|
321
|
+
* Builds one rendered managed-process definition without registering ownership.
|
|
299
322
|
* @param {import("./config.js").ProcessConfig} processConfig - Process config.
|
|
300
323
|
* @param {BuildProcessOptions} [options] - Build options.
|
|
301
|
-
* @returns {ManagedProcess} Managed process.
|
|
324
|
+
* @returns {ConstructorParameters<typeof ManagedProcess>[0] & import("./managed-process.js").ManagedProcessDefinition} Managed process definition.
|
|
302
325
|
*/
|
|
303
|
-
|
|
326
|
+
processDefinition(processConfig, options = {}) {
|
|
304
327
|
const index = options.index ?? 0
|
|
305
328
|
const count = options.count ?? 1
|
|
306
329
|
const instanceId = options.instanceId ?? processConfig.id
|
|
@@ -311,12 +334,12 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
311
334
|
...renderedEnv
|
|
312
335
|
}
|
|
313
336
|
|
|
314
|
-
|
|
337
|
+
return {
|
|
315
338
|
command: renderTemplate(processConfig.command, context),
|
|
316
339
|
cwd: processConfig.cwd ? renderTemplate(processConfig.cwd, context) : this.releasePath,
|
|
317
340
|
env: processEnv,
|
|
318
341
|
id: instanceId,
|
|
319
|
-
lifecycle: processConfig.lifecycle,
|
|
342
|
+
lifecycle: processConfig.lifecycle || {drainTimeoutMs: 0},
|
|
320
343
|
logger: (message, data = {}) => this.logger(message, {processId: instanceId, releaseId: this.releaseId, ...data}),
|
|
321
344
|
memory: processConfig.memory,
|
|
322
345
|
outputLines: processConfig.outputLines,
|
|
@@ -325,7 +348,18 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
325
348
|
shouldRestart: options.shouldRestart || (() => this.state === "active" || this.state === "starting"),
|
|
326
349
|
stopSignal: processConfig.stopSignal,
|
|
327
350
|
stopTimeoutMs: processConfig.gracefulStopMs
|
|
328
|
-
}
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
/**
|
|
355
|
+
* Builds a managed process from config.
|
|
356
|
+
* @param {import("./config.js").ProcessConfig} processConfig - Process config.
|
|
357
|
+
* @param {BuildProcessOptions} [options] - Build options.
|
|
358
|
+
* @returns {ManagedProcess} Managed process.
|
|
359
|
+
*/
|
|
360
|
+
buildProcess(processConfig, options = {}) {
|
|
361
|
+
const definition = this.processDefinition(processConfig, options)
|
|
362
|
+
const instanceId = options.instanceId ?? processConfig.id
|
|
329
363
|
|
|
330
364
|
return this.processFactory
|
|
331
365
|
? this.processFactory(options.guardianKey || `release:${this.releaseId}:${instanceId}`, definition)
|
|
@@ -347,7 +381,7 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
347
381
|
|
|
348
382
|
for (let index = 0; index < instances.length; index += 1) {
|
|
349
383
|
const instance = instances[index]
|
|
350
|
-
const nextDefinition = this.
|
|
384
|
+
const nextDefinition = this.processDefinition(processConfig, {
|
|
351
385
|
count: processConfig.replicas,
|
|
352
386
|
index,
|
|
353
387
|
instanceId: instance.id
|
|
@@ -543,21 +577,25 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
543
577
|
/**
|
|
544
578
|
* Marks the generation retired and quiesces its jobs-main and non-blocking workers as one unit.
|
|
545
579
|
* @param {import("./config.js").RollbridgeConfig} [config] - Refreshed retirement config.
|
|
580
|
+
* @param {{retry?: boolean}} [options] - Explicit durable-transition resume options.
|
|
546
581
|
* @returns {Promise<void>} Resolves when retirement quiescence succeeds.
|
|
547
582
|
*/
|
|
548
|
-
async beginRetirement(config = this.config) {
|
|
583
|
+
async beginRetirement(config = this.config, {retry = false} = {}) {
|
|
549
584
|
if (this.state === "draining") {
|
|
550
|
-
if (
|
|
551
|
-
|
|
585
|
+
if (!retry) {
|
|
586
|
+
if (this.retirementError) throw new Error(this.retirementError)
|
|
587
|
+
return
|
|
588
|
+
}
|
|
589
|
+
} else {
|
|
590
|
+
this.state = "draining"
|
|
591
|
+
this.drainStartedAt = new Date().toISOString()
|
|
552
592
|
}
|
|
553
593
|
|
|
554
|
-
this.state = "draining"
|
|
555
|
-
this.drainStartedAt = new Date().toISOString()
|
|
556
594
|
this.refreshProcessDefinitions(config)
|
|
557
595
|
const generationIds = new Set([...this.handoffServiceIds, ...this.nonBlockingDrainIds])
|
|
558
596
|
const results = await Promise.allSettled([...this.processes.entries()]
|
|
559
597
|
.filter(([id]) => generationIds.has(id))
|
|
560
|
-
.map(([, processInstance]) => processInstance.quiesceStrict()))
|
|
598
|
+
.map(([, processInstance]) => retry ? processInstance.requiesceStrict() : processInstance.quiesceStrict()))
|
|
561
599
|
const errors = results.filter((result) => result.status === "rejected").map((result) => result.reason)
|
|
562
600
|
|
|
563
601
|
if (errors.length > 0) {
|
|
@@ -565,6 +603,7 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
565
603
|
this.retirementError = `${failure.message}: ${errors.map((error) => error instanceof Error ? error.message : String(error)).join("; ")}`
|
|
566
604
|
throw failure
|
|
567
605
|
}
|
|
606
|
+
this.retirementError = undefined
|
|
568
607
|
}
|
|
569
608
|
|
|
570
609
|
/** @returns {Promise<void>} Stops all release-owned processes. */
|
|
@@ -204,6 +204,50 @@ test("validateConfig defaults lifecycle, accepts hooks, and rejects bad values",
|
|
|
204
204
|
assert.deepEqual(validateLifecycle({stopCommand: "kill -TERM $ROLLBRIDGE_PID"}).issues, [])
|
|
205
205
|
})
|
|
206
206
|
|
|
207
|
+
test("validateConfig accepts one durable handoff activation lifecycle and rejects unsafe placements", () => {
|
|
208
|
+
const base = {
|
|
209
|
+
application: "demo",
|
|
210
|
+
control: {path: "/tmp/demo.sock"},
|
|
211
|
+
ownerRecovery: {reconnectGraceMs: 30000},
|
|
212
|
+
processes: [
|
|
213
|
+
{command: "run web", id: "web", policy: "proxied", port: {from: 18000, to: 18099}},
|
|
214
|
+
{
|
|
215
|
+
command: "run jobs",
|
|
216
|
+
deployStrategy: "handoff",
|
|
217
|
+
id: "jobs",
|
|
218
|
+
lifecycle: {activateCommand: "jobs activate", quietCommand: "jobs retire"},
|
|
219
|
+
policy: "service",
|
|
220
|
+
port: {from: 18100, to: 18199}
|
|
221
|
+
}
|
|
222
|
+
],
|
|
223
|
+
proxy: {host: "127.0.0.1", port: 8182},
|
|
224
|
+
statePath: "/tmp/demo.state.json"
|
|
225
|
+
}
|
|
226
|
+
const valid = validateConfig(base)
|
|
227
|
+
|
|
228
|
+
assert.deepEqual(valid.issues, [])
|
|
229
|
+
assert.equal(valid.config.processes[1].lifecycle.activateCommand, "jobs activate")
|
|
230
|
+
|
|
231
|
+
const invalidType = validateConfig({...base, processes: [base.processes[0], {...base.processes[1], lifecycle: {activateCommand: 5, quietCommand: "jobs retire"}}]})
|
|
232
|
+
assert.ok(invalidType.issues.some((issue) => issue.message === "processes[1].lifecycle.activateCommand must be a string"))
|
|
233
|
+
|
|
234
|
+
const missingRetirement = validateConfig({...base, processes: [base.processes[0], {...base.processes[1], lifecycle: {activateCommand: "jobs activate"}}]})
|
|
235
|
+
assert.ok(missingRetirement.issues.some((issue) => /requires lifecycle\.quietCommand/.test(issue.message)))
|
|
236
|
+
|
|
237
|
+
const nonHandoff = validateConfig({...base, processes: [base.processes[0], {...base.processes[1], deployStrategy: "persistent"}]})
|
|
238
|
+
assert.ok(nonHandoff.issues.some((issue) => /activateCommand.*handoff service/.test(issue.message)))
|
|
239
|
+
|
|
240
|
+
const withoutRecovery = validateConfig({...base, ownerRecovery: undefined, statePath: undefined})
|
|
241
|
+
assert.ok(withoutRecovery.issues.some((issue) => /activateCommand requires ownerRecovery and statePath/.test(issue.message)))
|
|
242
|
+
|
|
243
|
+
const duplicate = validateConfig({...base, processes: [
|
|
244
|
+
base.processes[0],
|
|
245
|
+
base.processes[1],
|
|
246
|
+
{...base.processes[1], id: "jobs-secondary", port: {from: 18200, to: 18299}}
|
|
247
|
+
]})
|
|
248
|
+
assert.ok(duplicate.issues.some((issue) => /at most one lifecycle\.activateCommand/.test(issue.message)))
|
|
249
|
+
})
|
|
250
|
+
|
|
207
251
|
test("validateConfig accepts indefinite graceful stop windows", () => {
|
|
208
252
|
const {config, issues} = validateConfig({
|
|
209
253
|
application: "demo",
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import assert from "node:assert/strict"
|
|
4
4
|
import {spawn} from "node:child_process"
|
|
5
|
+
import {once} from "node:events"
|
|
5
6
|
import fs from "node:fs/promises"
|
|
6
7
|
import os from "node:os"
|
|
7
8
|
import path from "node:path"
|
|
@@ -49,6 +50,44 @@ test("guardian inventory removes only an exact owned provenance", async () => {
|
|
|
49
50
|
}
|
|
50
51
|
})
|
|
51
52
|
|
|
53
|
+
test("guardian runs a strict activation lifecycle command for the exact registered process", async () => {
|
|
54
|
+
const fixture = await createGuardian()
|
|
55
|
+
const activationPath = path.join(fixture.root, "activated")
|
|
56
|
+
const processInstance = fixture.client.process("candidate-activation", {
|
|
57
|
+
...definition("candidate-activation"),
|
|
58
|
+
lifecycle: {activateCommand: `printf activated > ${JSON.stringify(activationPath)}`, drainTimeoutMs: 0}
|
|
59
|
+
})
|
|
60
|
+
|
|
61
|
+
try {
|
|
62
|
+
await processInstance.start()
|
|
63
|
+
await processInstance.activateStrict()
|
|
64
|
+
assert.equal(await fs.readFile(activationPath, "utf8"), "activated")
|
|
65
|
+
} finally {
|
|
66
|
+
await cleanupGuardian(fixture)
|
|
67
|
+
}
|
|
68
|
+
})
|
|
69
|
+
|
|
70
|
+
test("guardian forwards each retained output line to its exact process proxy", async () => {
|
|
71
|
+
const fixture = await createGuardian()
|
|
72
|
+
const marker = "guardian-output-ready"
|
|
73
|
+
const processInstance = fixture.client.process("output", {
|
|
74
|
+
...definition("output"),
|
|
75
|
+
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`console.log(${JSON.stringify(marker)})`)}`
|
|
76
|
+
})
|
|
77
|
+
const logged = once(processInstance, "log")
|
|
78
|
+
const exitedFirst = once(processInstance, "exit").then(() => { throw new Error("Guardian process exited before forwarding retained output") })
|
|
79
|
+
|
|
80
|
+
try {
|
|
81
|
+
await processInstance.start()
|
|
82
|
+
const [entry] = await Promise.race([logged, exitedFirst])
|
|
83
|
+
|
|
84
|
+
assert.equal(entry.line, marker)
|
|
85
|
+
assert.ok(processInstance.status().logs.some((candidate) => candidate.line === marker))
|
|
86
|
+
} finally {
|
|
87
|
+
await cleanupGuardian(fixture)
|
|
88
|
+
}
|
|
89
|
+
})
|
|
90
|
+
|
|
52
91
|
test("guardian shutdown reports an exact owned process stop failure", async () => {
|
|
53
92
|
const fixture = await createGuardian()
|
|
54
93
|
const processInstance = fixture.client.process("broken-stop", {...definition("broken-stop"), stopSignal: "NOT_A_SIGNAL"})
|
|
@@ -171,6 +210,37 @@ test("replacement staging rejects owner state published after prepare", async ()
|
|
|
171
210
|
}
|
|
172
211
|
})
|
|
173
212
|
|
|
213
|
+
test("retired owner replacement requires unchanged authority and the exact control path absent", async () => {
|
|
214
|
+
const fixture = await createGuardian()
|
|
215
|
+
const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
216
|
+
const controlPath = path.join(fixture.root, "rollbridge.sock")
|
|
217
|
+
const authority = {configDigest: "incumbent", runtime: null}
|
|
218
|
+
const nextAuthority = {configDigest: "candidate", runtime: null}
|
|
219
|
+
const snapshot = {activeReleaseId: "v1", control: {path: controlPath}}
|
|
220
|
+
|
|
221
|
+
try {
|
|
222
|
+
await fixture.client.publishOwnerState({authority, snapshot})
|
|
223
|
+
await candidate.connect()
|
|
224
|
+
const changed = await candidate.prepareOwnerReplacement(authority, nextAuthority)
|
|
225
|
+
|
|
226
|
+
await candidate.stageOwnerReplacement(changed.replacementId, {authority: nextAuthority, snapshot})
|
|
227
|
+
await assert.rejects(() => candidate.commitRetiredOwnerReplacement(changed.replacementId), /unchanged owner authority/)
|
|
228
|
+
await candidate.abortOwnerReplacement(changed.replacementId)
|
|
229
|
+
|
|
230
|
+
const occupied = await candidate.prepareOwnerReplacement(authority, authority)
|
|
231
|
+
|
|
232
|
+
await candidate.stageOwnerReplacement(occupied.replacementId, {authority, snapshot})
|
|
233
|
+
await fs.writeFile(controlPath, "occupied\n")
|
|
234
|
+
await assert.rejects(() => candidate.commitRetiredOwnerReplacement(occupied.replacementId), /control socket .* still exists/)
|
|
235
|
+
await candidate.abortOwnerReplacement(occupied.replacementId)
|
|
236
|
+
await fixture.client.shutdown()
|
|
237
|
+
await fixture.client.guardianExit()
|
|
238
|
+
} finally {
|
|
239
|
+
candidate.disconnect()
|
|
240
|
+
await cleanupGuardian(fixture)
|
|
241
|
+
}
|
|
242
|
+
})
|
|
243
|
+
|
|
174
244
|
test("first upgrade migrates a real pre-split guardian without replacing its owned process", async () => {
|
|
175
245
|
const fixture = await createLegacyGuardian()
|
|
176
246
|
const processDefinition = definition("legacy-worker")
|