rollbridge 0.1.31 → 0.1.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +11 -6
- package/README.md +28 -12
- package/changelog.d/20260830-release-generation-activation-lifecycle.md +9 -0
- package/docs/cli.md +10 -7
- package/docs/config.md +41 -13
- package/docs/tensorbuzz-runbook.md +4 -2
- package/docs/velocious.md +19 -7
- package/docs/workers.md +18 -9
- package/examples/tensorbuzz.com.js +9 -5
- package/package.json +1 -1
- package/src/config.js +32 -2
- package/src/daemon.js +264 -20
- package/src/guardian-client.js +41 -3
- package/src/managed-process.js +85 -13
- package/src/process-guardian.js +13 -1
- package/src/release-group.js +53 -14
- package/test/config-validation.test.js +44 -0
- package/test/guardian-client.test.js +39 -0
- package/test/managed-process.test.js +33 -0
- package/test/owner-recovery.test.js +396 -6
- package/test/owner-replacement.test.js +107 -2
- package/test/rollbridge.test.js +333 -6
package/src/managed-process.js
CHANGED
|
@@ -4,15 +4,18 @@ import {EventEmitter} from "node:events"
|
|
|
4
4
|
import {spawn} from "node:child_process"
|
|
5
5
|
import {processGroupHasLiveMembers, processGroupMembers} from "./process-memory.js"
|
|
6
6
|
|
|
7
|
+
const ACTIVATION_HOOK_TIMEOUT_MS = 30000
|
|
8
|
+
|
|
7
9
|
/**
|
|
8
10
|
* @typedef {import("./json.js").JsonValue} JsonValue
|
|
9
11
|
* @typedef {"starting" | "running" | "quiesced" | "stopping" | "stopped" | "failed"} ManagedProcessState
|
|
10
12
|
* @typedef {"deploy" | "crash" | "manual" | "memory"} ManagedProcessStartReason
|
|
13
|
+
* @typedef {"active" | "candidate" | "retired"} LifecycleRole
|
|
11
14
|
* @typedef {import("node:child_process").ChildProcess["signalCode"]} ProcessExitSignal
|
|
12
15
|
* @typedef {{at: string, line: string, stream: "stdout" | "stderr"}} ManagedProcessLog
|
|
13
16
|
* @typedef {import("./config.js").StopTimeoutMs} StopTimeoutMs
|
|
14
17
|
* @typedef {{command: string, cwd: string | undefined, env: Record<string, string | undefined>, lifecycle: import("./config.js").LifecycleConfig, logger: (message: string, data?: Record<string, import("./json.js").JsonValue>) => void, memory: import("./config.js").MemoryConfig | undefined, outputLines: number, restart: import("./config.js").RestartConfig, restartDelayMs: number, shouldRestart: () => boolean, stopSignal: string, stopTimeoutMs: StopTimeoutMs}} ManagedProcessDefinition
|
|
15
|
-
* @typedef {{children: import("./process-memory.js").ProcessGroupMember[], command: string, cwd: string | undefined, exitCode: number | null | undefined, exitSignal: ProcessExitSignal | undefined, id: string, lastMemoryRestartAt: string | undefined, lastStartReason: ManagedProcessStartReason | undefined, logs: ManagedProcessLog[], memoryRestarts: number, pid: number | undefined, restarts: number, rssBytes: number | undefined, startedAt: string | undefined, state: ManagedProcessState, uptimeMs: number | undefined}} ManagedProcessStatus
|
|
18
|
+
* @typedef {{children: import("./process-memory.js").ProcessGroupMember[], command: string, cwd: string | undefined, exitCode: number | null | undefined, exitSignal: ProcessExitSignal | undefined, id: string, lastMemoryRestartAt: string | undefined, lastStartReason: ManagedProcessStartReason | undefined, lifecycleRole?: LifecycleRole, logs: ManagedProcessLog[], memoryRestarts: number, pid: number | undefined, restarts: number, rssBytes: number | undefined, startedAt: string | undefined, state: ManagedProcessState, uptimeMs: number | undefined}} ManagedProcessStatus
|
|
16
19
|
*/
|
|
17
20
|
|
|
18
21
|
export default class ManagedProcess extends EventEmitter {
|
|
@@ -62,6 +65,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
62
65
|
this.memoryWarned = false
|
|
63
66
|
this.startedAtMs = /** @type {number | undefined} */ (undefined)
|
|
64
67
|
this.intentionalStop = false
|
|
68
|
+
this.lifecycleRole = /** @type {LifecycleRole} */ ("candidate")
|
|
65
69
|
this.intentionalStopSignal = /** @type {ProcessExitSignal | undefined} */ (undefined)
|
|
66
70
|
this.quiescePromise = /** @type {Promise<void> | undefined} */ (undefined)
|
|
67
71
|
this.quiesceError = /** @type {Error | undefined} */ (undefined)
|
|
@@ -76,9 +80,11 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
76
80
|
|
|
77
81
|
/**
|
|
78
82
|
* @param {ManagedProcessStartReason} [reason] - Why the process is being started (deploy by default; "crash" on auto-restart, "manual" via the restart command).
|
|
79
|
-
* @
|
|
83
|
+
* @param {LifecycleRole} [lifecycleRole] - Exact desired role to restore before reporting the process running.
|
|
84
|
+
* @returns {Promise<void>} Resolves after spawn and lifecycle-role restoration.
|
|
80
85
|
*/
|
|
81
|
-
async start(reason = "deploy") {
|
|
86
|
+
async start(reason = "deploy", lifecycleRole) {
|
|
87
|
+
if (lifecycleRole) this.lifecycleRole = lifecycleRole
|
|
82
88
|
if (this.child) return
|
|
83
89
|
|
|
84
90
|
this.intentionalStop = false
|
|
@@ -109,16 +115,35 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
109
115
|
})
|
|
110
116
|
|
|
111
117
|
child.once("spawn", () => {
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
118
|
+
void (async () => {
|
|
119
|
+
this.startedAtMs = Date.now()
|
|
120
|
+
this.lastStartReason = reason
|
|
121
|
+
try {
|
|
122
|
+
await this.restoreLifecycleRole()
|
|
123
|
+
} catch (error) {
|
|
124
|
+
this.state = "failed"
|
|
125
|
+
this.logger("process lifecycle role restoration failed", {error: error instanceof Error ? error.message : String(error), id: this.id, role: this.lifecycleRole})
|
|
126
|
+
reject(error)
|
|
127
|
+
return
|
|
128
|
+
}
|
|
129
|
+
if (this.child !== child) {
|
|
130
|
+
reject(new Error(`Process ${this.id} exited before lifecycle role ${this.lifecycleRole} was restored`))
|
|
131
|
+
return
|
|
132
|
+
}
|
|
133
|
+
this.state = "running"
|
|
134
|
+
this.logger("process started", {command: this.command, id: this.id, pid: child.pid || null, reason})
|
|
135
|
+
this.startMemoryMonitor()
|
|
136
|
+
this.emit("started")
|
|
137
|
+
resolve(undefined)
|
|
138
|
+
})()
|
|
119
139
|
})
|
|
120
140
|
child.once("error", (error) => {
|
|
121
141
|
this.state = "failed"
|
|
142
|
+
if (this.child === child) {
|
|
143
|
+
this.child = undefined
|
|
144
|
+
this.pid = undefined
|
|
145
|
+
this.exitPromise = undefined
|
|
146
|
+
}
|
|
122
147
|
reject(error)
|
|
123
148
|
})
|
|
124
149
|
child.stdout.setEncoding("utf8")
|
|
@@ -157,11 +182,14 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
157
182
|
for (const line of String(chunk).split(/\r?\n/)) {
|
|
158
183
|
if (!line) continue
|
|
159
184
|
|
|
160
|
-
|
|
185
|
+
const entry = {at: new Date().toISOString(), line, stream}
|
|
186
|
+
|
|
187
|
+
this.logs.push(entry)
|
|
161
188
|
|
|
162
189
|
if (this.logs.length > this.outputLines) {
|
|
163
190
|
this.logs.splice(0, this.logs.length - this.outputLines)
|
|
164
191
|
}
|
|
192
|
+
this.emit("log", entry)
|
|
165
193
|
}
|
|
166
194
|
}
|
|
167
195
|
|
|
@@ -321,6 +349,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
321
349
|
if (this.memoryRestarting) return
|
|
322
350
|
|
|
323
351
|
this.memoryRestarting = true
|
|
352
|
+
const lifecycleRole = this.lifecycleRole
|
|
324
353
|
|
|
325
354
|
try {
|
|
326
355
|
await this.stop()
|
|
@@ -333,7 +362,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
333
362
|
this.memoryRestarts += 1
|
|
334
363
|
this.lastMemoryRestartAtMs = Date.now()
|
|
335
364
|
this.memoryWarned = false
|
|
336
|
-
await this.start("memory")
|
|
365
|
+
await this.start("memory", lifecycleRole)
|
|
337
366
|
} catch (error) {
|
|
338
367
|
this.logger("memory restart failed", {error: error instanceof Error ? error.message : String(error), id: this.id})
|
|
339
368
|
} finally {
|
|
@@ -415,11 +444,15 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
415
444
|
}
|
|
416
445
|
if (!this.child?.pid) {
|
|
417
446
|
this.state = "stopped"
|
|
447
|
+
if (this.lifecycle.activateCommand) this.lifecycleRole = "retired"
|
|
418
448
|
return
|
|
419
449
|
}
|
|
420
450
|
this.state = "stopping"
|
|
421
451
|
if (this.lifecycle.quietCommand) this.quiesceError = await this.runHook(this.lifecycle.quietCommand, this.hookTimeoutMs(), "quiet command")
|
|
422
|
-
if (!this.quiesceError)
|
|
452
|
+
if (!this.quiesceError) {
|
|
453
|
+
this.state = "quiesced"
|
|
454
|
+
if (this.lifecycle.activateCommand) this.lifecycleRole = "retired"
|
|
455
|
+
}
|
|
423
456
|
})()
|
|
424
457
|
return await this.quiescePromise
|
|
425
458
|
}
|
|
@@ -430,6 +463,44 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
430
463
|
if (this.quiesceError) throw this.quiesceError
|
|
431
464
|
}
|
|
432
465
|
|
|
466
|
+
/** Re-runs the idempotent quiet hook for an explicitly resumed durable transition. */
|
|
467
|
+
async requiesceStrict() {
|
|
468
|
+
this.quiescePromise = undefined
|
|
469
|
+
this.quiesceError = undefined
|
|
470
|
+
await this.quiesceStrict()
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
/** Runs the opt-in generation activation command and rejects on any hook failure. */
|
|
474
|
+
async activateStrict() {
|
|
475
|
+
const command = this.lifecycle.activateCommand
|
|
476
|
+
|
|
477
|
+
if (!command) return
|
|
478
|
+
const error = await this.runHook(command, ACTIVATION_HOOK_TIMEOUT_MS, "activate command", this.pid)
|
|
479
|
+
|
|
480
|
+
if (error) throw error
|
|
481
|
+
this.lifecycleRole = "active"
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
/**
|
|
485
|
+
* Records the durable desired role without firing a lifecycle command.
|
|
486
|
+
* @param {LifecycleRole} role - Exact role owned by this process generation.
|
|
487
|
+
*/
|
|
488
|
+
async setLifecycleRole(role) {
|
|
489
|
+
this.lifecycleRole = role
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
/** Restores an active or retired role after this exact process starts. */
|
|
493
|
+
async restoreLifecycleRole() {
|
|
494
|
+
if (!this.lifecycle.activateCommand || this.lifecycleRole === "candidate") return
|
|
495
|
+
const command = this.lifecycleRole === "active" ? this.lifecycle.activateCommand : this.lifecycle.quietCommand
|
|
496
|
+
|
|
497
|
+
if (!command) throw new Error(`Process ${this.id} cannot restore lifecycle role ${this.lifecycleRole} without its paired command`)
|
|
498
|
+
const timeoutMs = this.lifecycleRole === "active" ? ACTIVATION_HOOK_TIMEOUT_MS : this.hookTimeoutMs()
|
|
499
|
+
const error = await this.runHook(command, timeoutMs, `${this.lifecycleRole === "active" ? "activate" : "quiet"} command`, this.pid)
|
|
500
|
+
|
|
501
|
+
if (error) throw error
|
|
502
|
+
}
|
|
503
|
+
|
|
433
504
|
/** @returns {number} Timeout used for lifecycle hooks. */
|
|
434
505
|
hookTimeoutMs() {
|
|
435
506
|
if (this.stopTimeoutMs === "indefinite") return 30000
|
|
@@ -642,6 +713,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
642
713
|
id: this.id,
|
|
643
714
|
lastMemoryRestartAt: this.lastMemoryRestartAtMs === undefined ? undefined : new Date(this.lastMemoryRestartAtMs).toISOString(),
|
|
644
715
|
lastStartReason: this.lastStartReason,
|
|
716
|
+
...(this.lifecycle.activateCommand ? {lifecycleRole: this.lifecycleRole} : {}),
|
|
645
717
|
logs: this.logs.slice(-this.outputLines),
|
|
646
718
|
memoryRestarts: this.memoryRestarts,
|
|
647
719
|
pid: this.pid,
|
package/src/process-guardian.js
CHANGED
|
@@ -24,6 +24,7 @@ import ManagedProcess from "./managed-process.js"
|
|
|
24
24
|
* @property {import("./json.js").JsonValue} [authority] - Expected current authority.
|
|
25
25
|
* @property {import("./json.js").JsonValue} [nextAuthority] - Requested replacement authority.
|
|
26
26
|
* @property {import("./managed-process.js").ManagedProcessStartReason} [reason] - Start reason.
|
|
27
|
+
* @property {import("./managed-process.js").LifecycleRole} [lifecycleRole] - Desired generation role restored during start.
|
|
27
28
|
* @property {string} token - Authentication token.
|
|
28
29
|
*/
|
|
29
30
|
|
|
@@ -396,6 +397,9 @@ async function execute(request, socket) {
|
|
|
396
397
|
? legacyGuardian.process(request.key, managedDefinition)
|
|
397
398
|
: new ManagedProcess(managedDefinition)
|
|
398
399
|
|
|
400
|
+
managedProcess.on("log", (entry) => {
|
|
401
|
+
broadcast({entry, event: "process-log", key: request.key, status: managedProcess.status()})
|
|
402
|
+
})
|
|
399
403
|
if (recoversLegacyProcess && "recover" in managedProcess && typeof managedProcess.recover === "function") await managedProcess.recover()
|
|
400
404
|
|
|
401
405
|
record.process = managedProcess
|
|
@@ -413,10 +417,18 @@ async function execute(request, socket) {
|
|
|
413
417
|
|
|
414
418
|
if (request.command === "start") {
|
|
415
419
|
record.desired = true
|
|
416
|
-
await record.process.start(request.reason)
|
|
420
|
+
await record.process.start(request.reason, request.lifecycleRole)
|
|
421
|
+
} else if (request.command === "activate") {
|
|
422
|
+
await record.process.activateStrict()
|
|
417
423
|
} else if (request.command === "quiesce") {
|
|
418
424
|
record.desired = false
|
|
419
425
|
await record.process.quiesceStrict()
|
|
426
|
+
} else if (request.command === "requiesce") {
|
|
427
|
+
record.desired = false
|
|
428
|
+
await record.process.requiesceStrict()
|
|
429
|
+
} else if (request.command === "set-lifecycle-role") {
|
|
430
|
+
if (request.lifecycleRole !== "active" && request.lifecycleRole !== "candidate" && request.lifecycleRole !== "retired") throw new Error("Guardian lifecycle role is invalid")
|
|
431
|
+
await record.process.setLifecycleRole(request.lifecycleRole)
|
|
420
432
|
} else if (request.command === "stop") {
|
|
421
433
|
record.desired = false
|
|
422
434
|
await record.process.stop(request.options)
|
package/src/release-group.js
CHANGED
|
@@ -116,8 +116,9 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
116
116
|
/**
|
|
117
117
|
* Reconstructs this release around processes still owned by the durable guardian.
|
|
118
118
|
* @param {ReleaseStatus} snapshot - Persisted release snapshot.
|
|
119
|
+
* @param {{synchronizeLifecycleRole?: boolean}} [options] - Guardian role synchronization options.
|
|
119
120
|
*/
|
|
120
|
-
async restore(snapshot) {
|
|
121
|
+
async restore(snapshot, {synchronizeLifecycleRole = true} = {}) {
|
|
121
122
|
if (!snapshot || snapshot.releaseId !== this.releaseId || snapshot.releasePath !== this.releasePath || snapshot.revision !== this.revision) {
|
|
122
123
|
throw new Error(`Persisted release identity mismatch for ${this.releaseId}`)
|
|
123
124
|
}
|
|
@@ -175,6 +176,17 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
175
176
|
if (processConfig.nonBlockingDrain) this.nonBlockingDrainIds.add(processStatus.id)
|
|
176
177
|
if ("recover" in processInstance && typeof processInstance.recover === "function") await processInstance.recover()
|
|
177
178
|
}
|
|
179
|
+
if (synchronizeLifecycleRole) await this.synchronizeLifecycleRoles()
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/** Synchronizes only the activation owner's durable role without firing its hook. */
|
|
183
|
+
async synchronizeLifecycleRoles() {
|
|
184
|
+
const processConfig = this.config.processes.find((candidate) => candidate.lifecycle.activateCommand !== undefined)
|
|
185
|
+
|
|
186
|
+
if (!processConfig) return
|
|
187
|
+
const lifecycleRole = this.state === "active" ? "active" : this.state === "draining" ? "retired" : "candidate"
|
|
188
|
+
|
|
189
|
+
await Promise.all(this.getProcesses(processConfig.id).map(({process}) => process.setLifecycleRole(lifecycleRole)))
|
|
178
190
|
}
|
|
179
191
|
|
|
180
192
|
/**
|
|
@@ -253,6 +265,17 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
253
265
|
this.activatedAt = new Date().toISOString()
|
|
254
266
|
}
|
|
255
267
|
|
|
268
|
+
/** Runs the configured release-generation activation acknowledgement, if any. */
|
|
269
|
+
async activateGeneration() {
|
|
270
|
+
const processConfig = this.config.processes.find((candidate) => candidate.lifecycle.activateCommand !== undefined)
|
|
271
|
+
|
|
272
|
+
if (!processConfig) return
|
|
273
|
+
const [instance] = this.getProcesses(processConfig.id)
|
|
274
|
+
|
|
275
|
+
if (!instance) throw new Error(`Generation activation process ${processConfig.id} is not running for release ${this.releaseId}`)
|
|
276
|
+
await instance.process.activateStrict()
|
|
277
|
+
}
|
|
278
|
+
|
|
256
279
|
/** @returns {Promise<void>} Allocates all configured per-process ports. */
|
|
257
280
|
async allocatePorts() {
|
|
258
281
|
if (this.portsAllocated) return
|
|
@@ -295,12 +318,12 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
295
318
|
}
|
|
296
319
|
|
|
297
320
|
/**
|
|
298
|
-
* Builds
|
|
321
|
+
* Builds one rendered managed-process definition without registering ownership.
|
|
299
322
|
* @param {import("./config.js").ProcessConfig} processConfig - Process config.
|
|
300
323
|
* @param {BuildProcessOptions} [options] - Build options.
|
|
301
|
-
* @returns {ManagedProcess} Managed process.
|
|
324
|
+
* @returns {ConstructorParameters<typeof ManagedProcess>[0] & import("./managed-process.js").ManagedProcessDefinition} Managed process definition.
|
|
302
325
|
*/
|
|
303
|
-
|
|
326
|
+
processDefinition(processConfig, options = {}) {
|
|
304
327
|
const index = options.index ?? 0
|
|
305
328
|
const count = options.count ?? 1
|
|
306
329
|
const instanceId = options.instanceId ?? processConfig.id
|
|
@@ -311,12 +334,12 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
311
334
|
...renderedEnv
|
|
312
335
|
}
|
|
313
336
|
|
|
314
|
-
|
|
337
|
+
return {
|
|
315
338
|
command: renderTemplate(processConfig.command, context),
|
|
316
339
|
cwd: processConfig.cwd ? renderTemplate(processConfig.cwd, context) : this.releasePath,
|
|
317
340
|
env: processEnv,
|
|
318
341
|
id: instanceId,
|
|
319
|
-
lifecycle: processConfig.lifecycle,
|
|
342
|
+
lifecycle: processConfig.lifecycle || {drainTimeoutMs: 0},
|
|
320
343
|
logger: (message, data = {}) => this.logger(message, {processId: instanceId, releaseId: this.releaseId, ...data}),
|
|
321
344
|
memory: processConfig.memory,
|
|
322
345
|
outputLines: processConfig.outputLines,
|
|
@@ -325,7 +348,18 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
325
348
|
shouldRestart: options.shouldRestart || (() => this.state === "active" || this.state === "starting"),
|
|
326
349
|
stopSignal: processConfig.stopSignal,
|
|
327
350
|
stopTimeoutMs: processConfig.gracefulStopMs
|
|
328
|
-
}
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
/**
|
|
355
|
+
* Builds a managed process from config.
|
|
356
|
+
* @param {import("./config.js").ProcessConfig} processConfig - Process config.
|
|
357
|
+
* @param {BuildProcessOptions} [options] - Build options.
|
|
358
|
+
* @returns {ManagedProcess} Managed process.
|
|
359
|
+
*/
|
|
360
|
+
buildProcess(processConfig, options = {}) {
|
|
361
|
+
const definition = this.processDefinition(processConfig, options)
|
|
362
|
+
const instanceId = options.instanceId ?? processConfig.id
|
|
329
363
|
|
|
330
364
|
return this.processFactory
|
|
331
365
|
? this.processFactory(options.guardianKey || `release:${this.releaseId}:${instanceId}`, definition)
|
|
@@ -347,7 +381,7 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
347
381
|
|
|
348
382
|
for (let index = 0; index < instances.length; index += 1) {
|
|
349
383
|
const instance = instances[index]
|
|
350
|
-
const nextDefinition = this.
|
|
384
|
+
const nextDefinition = this.processDefinition(processConfig, {
|
|
351
385
|
count: processConfig.replicas,
|
|
352
386
|
index,
|
|
353
387
|
instanceId: instance.id
|
|
@@ -543,21 +577,25 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
543
577
|
/**
|
|
544
578
|
* Marks the generation retired and quiesces its jobs-main and non-blocking workers as one unit.
|
|
545
579
|
* @param {import("./config.js").RollbridgeConfig} [config] - Refreshed retirement config.
|
|
580
|
+
* @param {{retry?: boolean}} [options] - Explicit durable-transition resume options.
|
|
546
581
|
* @returns {Promise<void>} Resolves when retirement quiescence succeeds.
|
|
547
582
|
*/
|
|
548
|
-
async beginRetirement(config = this.config) {
|
|
583
|
+
async beginRetirement(config = this.config, {retry = false} = {}) {
|
|
549
584
|
if (this.state === "draining") {
|
|
550
|
-
if (
|
|
551
|
-
|
|
585
|
+
if (!retry) {
|
|
586
|
+
if (this.retirementError) throw new Error(this.retirementError)
|
|
587
|
+
return
|
|
588
|
+
}
|
|
589
|
+
} else {
|
|
590
|
+
this.state = "draining"
|
|
591
|
+
this.drainStartedAt = new Date().toISOString()
|
|
552
592
|
}
|
|
553
593
|
|
|
554
|
-
this.state = "draining"
|
|
555
|
-
this.drainStartedAt = new Date().toISOString()
|
|
556
594
|
this.refreshProcessDefinitions(config)
|
|
557
595
|
const generationIds = new Set([...this.handoffServiceIds, ...this.nonBlockingDrainIds])
|
|
558
596
|
const results = await Promise.allSettled([...this.processes.entries()]
|
|
559
597
|
.filter(([id]) => generationIds.has(id))
|
|
560
|
-
.map(([, processInstance]) => processInstance.quiesceStrict()))
|
|
598
|
+
.map(([, processInstance]) => retry ? processInstance.requiesceStrict() : processInstance.quiesceStrict()))
|
|
561
599
|
const errors = results.filter((result) => result.status === "rejected").map((result) => result.reason)
|
|
562
600
|
|
|
563
601
|
if (errors.length > 0) {
|
|
@@ -565,6 +603,7 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
565
603
|
this.retirementError = `${failure.message}: ${errors.map((error) => error instanceof Error ? error.message : String(error)).join("; ")}`
|
|
566
604
|
throw failure
|
|
567
605
|
}
|
|
606
|
+
this.retirementError = undefined
|
|
568
607
|
}
|
|
569
608
|
|
|
570
609
|
/** @returns {Promise<void>} Stops all release-owned processes. */
|
|
@@ -204,6 +204,50 @@ test("validateConfig defaults lifecycle, accepts hooks, and rejects bad values",
|
|
|
204
204
|
assert.deepEqual(validateLifecycle({stopCommand: "kill -TERM $ROLLBRIDGE_PID"}).issues, [])
|
|
205
205
|
})
|
|
206
206
|
|
|
207
|
+
test("validateConfig accepts one durable handoff activation lifecycle and rejects unsafe placements", () => {
|
|
208
|
+
const base = {
|
|
209
|
+
application: "demo",
|
|
210
|
+
control: {path: "/tmp/demo.sock"},
|
|
211
|
+
ownerRecovery: {reconnectGraceMs: 30000},
|
|
212
|
+
processes: [
|
|
213
|
+
{command: "run web", id: "web", policy: "proxied", port: {from: 18000, to: 18099}},
|
|
214
|
+
{
|
|
215
|
+
command: "run jobs",
|
|
216
|
+
deployStrategy: "handoff",
|
|
217
|
+
id: "jobs",
|
|
218
|
+
lifecycle: {activateCommand: "jobs activate", quietCommand: "jobs retire"},
|
|
219
|
+
policy: "service",
|
|
220
|
+
port: {from: 18100, to: 18199}
|
|
221
|
+
}
|
|
222
|
+
],
|
|
223
|
+
proxy: {host: "127.0.0.1", port: 8182},
|
|
224
|
+
statePath: "/tmp/demo.state.json"
|
|
225
|
+
}
|
|
226
|
+
const valid = validateConfig(base)
|
|
227
|
+
|
|
228
|
+
assert.deepEqual(valid.issues, [])
|
|
229
|
+
assert.equal(valid.config.processes[1].lifecycle.activateCommand, "jobs activate")
|
|
230
|
+
|
|
231
|
+
const invalidType = validateConfig({...base, processes: [base.processes[0], {...base.processes[1], lifecycle: {activateCommand: 5, quietCommand: "jobs retire"}}]})
|
|
232
|
+
assert.ok(invalidType.issues.some((issue) => issue.message === "processes[1].lifecycle.activateCommand must be a string"))
|
|
233
|
+
|
|
234
|
+
const missingRetirement = validateConfig({...base, processes: [base.processes[0], {...base.processes[1], lifecycle: {activateCommand: "jobs activate"}}]})
|
|
235
|
+
assert.ok(missingRetirement.issues.some((issue) => /requires lifecycle\.quietCommand/.test(issue.message)))
|
|
236
|
+
|
|
237
|
+
const nonHandoff = validateConfig({...base, processes: [base.processes[0], {...base.processes[1], deployStrategy: "persistent"}]})
|
|
238
|
+
assert.ok(nonHandoff.issues.some((issue) => /activateCommand.*handoff service/.test(issue.message)))
|
|
239
|
+
|
|
240
|
+
const withoutRecovery = validateConfig({...base, ownerRecovery: undefined, statePath: undefined})
|
|
241
|
+
assert.ok(withoutRecovery.issues.some((issue) => /activateCommand requires ownerRecovery and statePath/.test(issue.message)))
|
|
242
|
+
|
|
243
|
+
const duplicate = validateConfig({...base, processes: [
|
|
244
|
+
base.processes[0],
|
|
245
|
+
base.processes[1],
|
|
246
|
+
{...base.processes[1], id: "jobs-secondary", port: {from: 18200, to: 18299}}
|
|
247
|
+
]})
|
|
248
|
+
assert.ok(duplicate.issues.some((issue) => /at most one lifecycle\.activateCommand/.test(issue.message)))
|
|
249
|
+
})
|
|
250
|
+
|
|
207
251
|
test("validateConfig accepts indefinite graceful stop windows", () => {
|
|
208
252
|
const {config, issues} = validateConfig({
|
|
209
253
|
application: "demo",
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import assert from "node:assert/strict"
|
|
4
4
|
import {spawn} from "node:child_process"
|
|
5
|
+
import {once} from "node:events"
|
|
5
6
|
import fs from "node:fs/promises"
|
|
6
7
|
import os from "node:os"
|
|
7
8
|
import path from "node:path"
|
|
@@ -49,6 +50,44 @@ test("guardian inventory removes only an exact owned provenance", async () => {
|
|
|
49
50
|
}
|
|
50
51
|
})
|
|
51
52
|
|
|
53
|
+
test("guardian runs a strict activation lifecycle command for the exact registered process", async () => {
|
|
54
|
+
const fixture = await createGuardian()
|
|
55
|
+
const activationPath = path.join(fixture.root, "activated")
|
|
56
|
+
const processInstance = fixture.client.process("candidate-activation", {
|
|
57
|
+
...definition("candidate-activation"),
|
|
58
|
+
lifecycle: {activateCommand: `printf activated > ${JSON.stringify(activationPath)}`, drainTimeoutMs: 0}
|
|
59
|
+
})
|
|
60
|
+
|
|
61
|
+
try {
|
|
62
|
+
await processInstance.start()
|
|
63
|
+
await processInstance.activateStrict()
|
|
64
|
+
assert.equal(await fs.readFile(activationPath, "utf8"), "activated")
|
|
65
|
+
} finally {
|
|
66
|
+
await cleanupGuardian(fixture)
|
|
67
|
+
}
|
|
68
|
+
})
|
|
69
|
+
|
|
70
|
+
test("guardian forwards each retained output line to its exact process proxy", async () => {
|
|
71
|
+
const fixture = await createGuardian()
|
|
72
|
+
const marker = "guardian-output-ready"
|
|
73
|
+
const processInstance = fixture.client.process("output", {
|
|
74
|
+
...definition("output"),
|
|
75
|
+
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`console.log(${JSON.stringify(marker)})`)}`
|
|
76
|
+
})
|
|
77
|
+
const logged = once(processInstance, "log")
|
|
78
|
+
const exitedFirst = once(processInstance, "exit").then(() => { throw new Error("Guardian process exited before forwarding retained output") })
|
|
79
|
+
|
|
80
|
+
try {
|
|
81
|
+
await processInstance.start()
|
|
82
|
+
const [entry] = await Promise.race([logged, exitedFirst])
|
|
83
|
+
|
|
84
|
+
assert.equal(entry.line, marker)
|
|
85
|
+
assert.ok(processInstance.status().logs.some((candidate) => candidate.line === marker))
|
|
86
|
+
} finally {
|
|
87
|
+
await cleanupGuardian(fixture)
|
|
88
|
+
}
|
|
89
|
+
})
|
|
90
|
+
|
|
52
91
|
test("guardian shutdown reports an exact owned process stop failure", async () => {
|
|
53
92
|
const fixture = await createGuardian()
|
|
54
93
|
const processInstance = fixture.client.process("broken-stop", {...definition("broken-stop"), stopSignal: "NOT_A_SIGNAL"})
|
|
@@ -66,6 +66,21 @@ test("keeps every output line when fewer than the retention limit are produced",
|
|
|
66
66
|
assert.deepEqual(logs.map((entry) => entry.line), ["one", "two"])
|
|
67
67
|
})
|
|
68
68
|
|
|
69
|
+
test("emits each output line after retaining it", () => {
|
|
70
|
+
const managed = buildProcess(50)
|
|
71
|
+
let observed
|
|
72
|
+
|
|
73
|
+
managed.once("log", (entry) => {
|
|
74
|
+
observed = {entry, retained: managed.status().logs}
|
|
75
|
+
})
|
|
76
|
+
managed.appendLog("stdout", "ready\n")
|
|
77
|
+
|
|
78
|
+
assert.deepEqual(observed, {
|
|
79
|
+
entry: managed.status().logs[0],
|
|
80
|
+
retained: managed.status().logs
|
|
81
|
+
})
|
|
82
|
+
})
|
|
83
|
+
|
|
69
84
|
test("reports zeroed restart and uptime fields before the process starts", () => {
|
|
70
85
|
const status = buildProcess(50).status()
|
|
71
86
|
|
|
@@ -385,6 +400,24 @@ test("a hanging lifecycle hook is bounded so stop still completes", async () =>
|
|
|
385
400
|
}
|
|
386
401
|
})
|
|
387
402
|
|
|
403
|
+
test("activateStrict runs the configured activation command once per call and rejects failures", async () => {
|
|
404
|
+
const commands = /** @type {{command: string, label: string, pid: number | undefined, timeoutMs: number}[]} */ ([])
|
|
405
|
+
const managed = buildLongLived(() => false)
|
|
406
|
+
|
|
407
|
+
managed.lifecycle = {activateCommand: "jobs activate", drainTimeoutMs: 0}
|
|
408
|
+
managed.pid = 4321
|
|
409
|
+
managed.runHook = async (command, timeoutMs, label, pid) => {
|
|
410
|
+
commands.push({command, label, pid, timeoutMs})
|
|
411
|
+
return undefined
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
await managed.activateStrict()
|
|
415
|
+
assert.deepEqual(commands, [{command: "jobs activate", label: "activate command", pid: 4321, timeoutMs: 30000}])
|
|
416
|
+
|
|
417
|
+
managed.runHook = async () => new Error("activation rejected")
|
|
418
|
+
await assert.rejects(() => managed.activateStrict(), /activation rejected/)
|
|
419
|
+
})
|
|
420
|
+
|
|
388
421
|
test("sends the configured stopSignal as the graceful stop signal", async () => {
|
|
389
422
|
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-stop-signal-"))
|
|
390
423
|
const readyPath = path.join(dir, "ready")
|