rollbridge 0.1.40 → 0.1.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/docs/config.md +4 -3
- package/examples/tensorbuzz.com.js +5 -2
- package/package.json +1 -1
- package/src/cli.js +25 -0
- package/src/config.js +32 -5
- package/src/daemon.js +180 -3
- package/src/guardian-client.js +55 -2
- package/src/managed-process.js +71 -11
- package/src/process-guardian.js +4 -1
- package/src/release-group.js +22 -0
- package/test/completion.test.js +4 -2
- package/test/config-examples.test.js +1 -0
- package/test/config-validation.test.js +23 -3
- package/test/guardian-client.test.js +82 -1
- package/test/managed-process.test.js +29 -2
- package/test/owner-recovery.test.js +11 -18
- package/test/owner-replacement.test.js +13 -19
- package/test/rollbridge.test.js +242 -13
package/README.md
CHANGED
|
@@ -89,6 +89,7 @@ export default {
|
|
|
89
89
|
command: "env VELOCIOUS_BACKGROUND_JOBS_PORT={{port}} npx velocious background-jobs-main",
|
|
90
90
|
lifecycle: {
|
|
91
91
|
activateCommand: 'npx velocious background-jobs:activate --generation "$ROLLBRIDGE_RELEASE_ID" --socket "$VELOCIOUS_BACKGROUND_JOBS_LIFECYCLE_SOCKET"',
|
|
92
|
+
activateTimeoutMs: 60000,
|
|
92
93
|
quietCommand: 'npx velocious background-jobs:retire --generation "$ROLLBRIDGE_RELEASE_ID" --socket "$VELOCIOUS_BACKGROUND_JOBS_LIFECYCLE_SOCKET"'
|
|
93
94
|
},
|
|
94
95
|
port: {from: 7331, to: 7399}
|
|
@@ -188,7 +189,8 @@ generation-scoped and resumable; failures remain visible and block unrelated
|
|
|
188
189
|
deploys. Post-commit singleton replacement is also journaled and must complete
|
|
189
190
|
before an exact retry reports success. If the active coordinator restarts,
|
|
190
191
|
Rollbridge restores its active role with the same bounded, generation-scoped
|
|
191
|
-
activation command before reporting it running.
|
|
192
|
+
activation command before reporting it running. Set `activateTimeoutMs` when the
|
|
193
|
+
activation acknowledgement can exceed its 30-second default. Omit `activateCommand` to
|
|
192
194
|
preserve the existing hook-free ordering.
|
|
193
195
|
|
|
194
196
|
See [`docs/workers.md`](docs/workers.md) for the full release-generation
|
package/docs/config.md
CHANGED
|
@@ -303,8 +303,8 @@ service when the service starts as a quiescent candidate and requires an explici
|
|
|
303
303
|
generation transition. Rollbridge starts and health-checks the complete candidate,
|
|
304
304
|
waits for the old generation's strict retirement acknowledgement, waits for the
|
|
305
305
|
candidate's strict activation acknowledgement, then commits the active release and
|
|
306
|
-
proxy target synchronously. Activation is
|
|
307
|
-
retirement uses the process's `gracefulStopMs` bound (or 30 seconds when that
|
|
306
|
+
proxy target synchronously. Activation is bounded by `lifecycle.activateTimeoutMs`,
|
|
307
|
+
which defaults to 30 seconds; retirement uses the process's `gracefulStopMs` bound (or 30 seconds when that
|
|
308
308
|
window is `"indefinite"`). Both run with the process environment plus
|
|
309
309
|
`ROLLBRIDGE_PID`.
|
|
310
310
|
|
|
@@ -347,7 +347,8 @@ legitimate hours-long generation drains are valid.
|
|
|
347
347
|
|
|
348
348
|
| Field | Type | Default | Description |
|
|
349
349
|
| --- | --- | --- | --- |
|
|
350
|
-
| `lifecycle.activateCommand` | string | unset | For one handoff service, acknowledge activation of its already-started candidate generation after the previous generation has acknowledged retirement. Requires `quietCommand`, `statePath`, and `ownerRecovery`; bounded
|
|
350
|
+
| `lifecycle.activateCommand` | string | unset | For one handoff service, acknowledge activation of its already-started candidate generation after the previous generation has acknowledged retirement. Requires `quietCommand`, `statePath`, and `ownerRecovery`; bounded by `activateTimeoutMs`. |
|
|
351
|
+
| `lifecycle.activateTimeoutMs` | positive number | `30000` | Bounds activation, reactivation, and active-role restoration commands. |
|
|
351
352
|
| `lifecycle.quietCommand` | string | unset | Run first to tell the process to stop accepting new work. Bounded by `gracefulStopMs`, or 30 seconds when that window is `"indefinite"`. |
|
|
352
353
|
| `lifecycle.drainCommand` | string | unset | Run after quieting to wait until the process has drained (it blocks until done). When unset, Rollbridge instead waits up to `drainTimeoutMs` for the process to exit on its own. Requires a positive `drainTimeoutMs` (which bounds it). |
|
|
353
354
|
| `lifecycle.drainTimeoutMs` | non-negative number | `0` | Bounds the drain step. `0` **skips the drain step entirely** (no `drainCommand`, no wait). |
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
// binds that stable HTTP port, forwards to the active release's internal web
|
|
5
5
|
// port and keeps Beacon daemon-wide. The lifecycle socket path must match the
|
|
6
6
|
// reviewed release-local Velocious jobs-main configuration; the worker appctl
|
|
7
|
-
//
|
|
7
|
+
// commands remain illustrative application-specific quiescence/reactivation controls.
|
|
8
8
|
|
|
9
9
|
export default {
|
|
10
10
|
application: "tensorbuzz",
|
|
@@ -68,7 +68,10 @@ export default {
|
|
|
68
68
|
VELOCIOUS_BACKGROUND_JOBS_PORT: "{{ports.background-jobs-main}}"
|
|
69
69
|
},
|
|
70
70
|
command: "wait-for-it 127.0.0.1:{{ports.beacon}} --strict -- wait-for-it 127.0.0.1:{{ports.background-jobs-main}} --strict -- npx velocious background-jobs-worker",
|
|
71
|
-
lifecycle: {
|
|
71
|
+
lifecycle: {
|
|
72
|
+
quietCommand: "appctl jobs-worker-retire --pid $ROLLBRIDGE_PID",
|
|
73
|
+
reactivateCommand: "appctl jobs-worker-reactivate --pid $ROLLBRIDGE_PID"
|
|
74
|
+
},
|
|
72
75
|
nonBlockingDrain: true,
|
|
73
76
|
gracefulStopMs: "indefinite"
|
|
74
77
|
},
|
package/package.json
CHANGED
package/src/cli.js
CHANGED
|
@@ -213,6 +213,31 @@ export async function runCli(argv) {
|
|
|
213
213
|
console.log(JSON.stringify(response, null, 2))
|
|
214
214
|
})
|
|
215
215
|
|
|
216
|
+
program
|
|
217
|
+
.command("recover-generation-transition")
|
|
218
|
+
.description("Restore the exact incumbent and retire a failed pre-commit generation candidate.")
|
|
219
|
+
.option("-c, --config <path>", "Config file path (defaults to rollbridge.js)")
|
|
220
|
+
.requiredOption("--release-path <path>", "Failed candidate release path")
|
|
221
|
+
.requiredOption("--release-id <id>", "Failed candidate release id")
|
|
222
|
+
.requiredOption("--revision <sha>", "Failed candidate revision")
|
|
223
|
+
.requiredOption("--previous-release-id <id>", "Expected authoritative incumbent release id")
|
|
224
|
+
.action(async (options) => {
|
|
225
|
+
const configPath = await resolveConfigPath(options.config)
|
|
226
|
+
const config = await loadConfig(configPath)
|
|
227
|
+
const response = await sendControlCommand({
|
|
228
|
+
command: {
|
|
229
|
+
command: "recover-generation-transition",
|
|
230
|
+
previousReleaseId: options.previousReleaseId,
|
|
231
|
+
releaseId: options.releaseId,
|
|
232
|
+
releasePath: options.releasePath,
|
|
233
|
+
revision: options.revision
|
|
234
|
+
},
|
|
235
|
+
path: config.control.path
|
|
236
|
+
})
|
|
237
|
+
|
|
238
|
+
console.log(JSON.stringify(response, null, 2))
|
|
239
|
+
})
|
|
240
|
+
|
|
216
241
|
program
|
|
217
242
|
.command("ensure-daemon")
|
|
218
243
|
.description("Start the daemon if the control socket is not already accepting commands.")
|
package/src/config.js
CHANGED
|
@@ -14,7 +14,7 @@ import {pathToFileURL} from "node:url"
|
|
|
14
14
|
* @typedef {"proxied" | "companion" | "singleton" | "service"} ProcessPolicy
|
|
15
15
|
* @typedef {{backoffFactor: number, maxDelayMs: number, maxRestarts: number | undefined, windowMs: number}} RestartConfig
|
|
16
16
|
* @typedef {{checkIntervalMs: number, limitBytes: number, warnBytes: number}} MemoryConfig
|
|
17
|
-
* @typedef {{activateCommand?: string, drainCommand?: string, drainTimeoutMs: number, quietCommand?: string, stopCommand?: string}} LifecycleConfig
|
|
17
|
+
* @typedef {{activateCommand?: string, activateTimeoutMs?: number, drainCommand?: string, drainTimeoutMs: number, quietCommand?: string, reactivateCommand?: string, stopCommand?: string}} LifecycleConfig
|
|
18
18
|
* @typedef {number | "indefinite"} StopTimeoutMs
|
|
19
19
|
* @typedef {"persistent" | "handoff"} ServiceDeployStrategy
|
|
20
20
|
* @typedef {{cwd?: string, deployStrategy: ServiceDeployStrategy, env: Record<string, string>, gracefulStopMs: StopTimeoutMs, health?: HealthConfig, id: string, lifecycle: LifecycleConfig, memory?: MemoryConfig, nonBlockingDrain: boolean, outputLines: number, policy: ProcessPolicy, port?: PortRange, replicas: number, restart: RestartConfig, restartDelayMs: number, stopSignal: string, command: string}} ProcessConfig
|
|
@@ -343,20 +343,29 @@ function normalizeMemory(value, key, issues) {
|
|
|
343
343
|
* @returns {LifecycleConfig} Normalized lifecycle hooks (no commands and a 0 drain by default).
|
|
344
344
|
*/
|
|
345
345
|
function normalizeLifecycle(value, key, issues) {
|
|
346
|
-
if (value === undefined || value === null) return {drainTimeoutMs: 0}
|
|
346
|
+
if (value === undefined || value === null) return {activateTimeoutMs: 30000, drainTimeoutMs: 0}
|
|
347
347
|
|
|
348
348
|
if (!isPlainObject(value)) {
|
|
349
|
-
issues.push({fix: `Set ${key} to a mapping with optional activateCommand, quietCommand, drainCommand, stopCommand, and drainTimeoutMs.`, message: `${key} must be an object`})
|
|
349
|
+
issues.push({fix: `Set ${key} to a mapping with optional activateCommand, quietCommand, reactivateCommand, drainCommand, stopCommand, and drainTimeoutMs.`, message: `${key} must be an object`})
|
|
350
350
|
|
|
351
|
-
return {drainTimeoutMs: 0}
|
|
351
|
+
return {activateTimeoutMs: 30000, drainTimeoutMs: 0}
|
|
352
352
|
}
|
|
353
353
|
|
|
354
|
+
const activateTimeoutMs = normalizeNumber(value.activateTimeoutMs, `${key}.activateTimeoutMs`, issues, {default: 30000})
|
|
354
355
|
const drainTimeoutMs = normalizeNumber(value.drainTimeoutMs, `${key}.drainTimeoutMs`, issues, {default: 0})
|
|
355
356
|
/** @type {LifecycleConfig} */
|
|
356
|
-
const lifecycle = {
|
|
357
|
+
const lifecycle = {
|
|
358
|
+
activateTimeoutMs: activateTimeoutMs > 0 ? activateTimeoutMs : 30000,
|
|
359
|
+
drainTimeoutMs: nonNegativeOrDefault(drainTimeoutMs, `${key}.drainTimeoutMs`, issues, 0, false)
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
if (activateTimeoutMs <= 0) {
|
|
363
|
+
issues.push({fix: `Set ${key}.activateTimeoutMs to a positive number of milliseconds, e.g. 30000.`, message: `${key}.activateTimeoutMs must be a positive number`})
|
|
364
|
+
}
|
|
357
365
|
|
|
358
366
|
if (value.activateCommand !== undefined) lifecycle.activateCommand = normalizeString(value.activateCommand, `${key}.activateCommand`, issues, {nonEmpty: true})
|
|
359
367
|
if (value.quietCommand !== undefined) lifecycle.quietCommand = normalizeString(value.quietCommand, `${key}.quietCommand`, issues, {nonEmpty: true})
|
|
368
|
+
if (value.reactivateCommand !== undefined) lifecycle.reactivateCommand = normalizeString(value.reactivateCommand, `${key}.reactivateCommand`, issues, {nonEmpty: true})
|
|
360
369
|
if (value.drainCommand !== undefined) lifecycle.drainCommand = normalizeString(value.drainCommand, `${key}.drainCommand`, issues, {nonEmpty: true})
|
|
361
370
|
if (value.stopCommand !== undefined) lifecycle.stopCommand = normalizeString(value.stopCommand, `${key}.stopCommand`, issues, {nonEmpty: true})
|
|
362
371
|
|
|
@@ -376,6 +385,7 @@ function normalizeLifecycle(value, key, issues) {
|
|
|
376
385
|
*/
|
|
377
386
|
function validateActivationLifecycle(processes, ownerRecovery, statePath, issues) {
|
|
378
387
|
const activated = processes.filter((processConfig) => processConfig.lifecycle.activateCommand !== undefined)
|
|
388
|
+
const reactivated = processes.filter((processConfig) => processConfig.lifecycle.reactivateCommand !== undefined)
|
|
379
389
|
|
|
380
390
|
if (activated.length > 1) {
|
|
381
391
|
issues.push({fix: "Configure lifecycle.activateCommand on only one release-generation coordinator.", message: "Config may define at most one lifecycle.activateCommand"})
|
|
@@ -390,6 +400,23 @@ function validateActivationLifecycle(processes, ownerRecovery, statePath, issues
|
|
|
390
400
|
}
|
|
391
401
|
}
|
|
392
402
|
|
|
403
|
+
for (const processConfig of reactivated) {
|
|
404
|
+
if (processConfig.policy !== "companion" || !processConfig.nonBlockingDrain) {
|
|
405
|
+
issues.push({fix: `Set lifecycle.reactivateCommand only on a nonBlockingDrain companion; "${processConfig.id}" is ${processConfig.policy} with nonBlockingDrain: ${processConfig.nonBlockingDrain}.`, message: `Process "${processConfig.id}" can only set lifecycle.reactivateCommand on a nonBlockingDrain companion`})
|
|
406
|
+
}
|
|
407
|
+
if (!processConfig.lifecycle.quietCommand) {
|
|
408
|
+
issues.push({fix: `Add lifecycle.quietCommand to "${processConfig.id}" so worker reactivation has a paired retirement command.`, message: `Process "${processConfig.id}" lifecycle.reactivateCommand requires lifecycle.quietCommand`})
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
if (activated.length > 0) {
|
|
413
|
+
for (const processConfig of processes) {
|
|
414
|
+
if (processConfig.nonBlockingDrain && processConfig.lifecycle.quietCommand && !processConfig.lifecycle.reactivateCommand) {
|
|
415
|
+
issues.push({fix: `Add lifecycle.reactivateCommand to "${processConfig.id}" so failed candidate recovery reverses its external quiet hook.`, message: `Process "${processConfig.id}" lifecycle.quietCommand requires lifecycle.reactivateCommand during generation activation recovery`})
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
|
|
393
420
|
if (activated.length > 0 && (!ownerRecovery || !statePath)) {
|
|
394
421
|
issues.push({fix: "Configure statePath and ownerRecovery for durable release-generation transition recovery.", message: "lifecycle.activateCommand requires ownerRecovery and statePath"})
|
|
395
422
|
}
|
package/src/daemon.js
CHANGED
|
@@ -24,8 +24,8 @@ const STATE_PERSIST_INTERVAL_MS = 5000
|
|
|
24
24
|
* @typedef {{attestation?: string, releaseId: string, releasePath: string, revision: string}} BootstrapIdentity
|
|
25
25
|
* @typedef {{id: string, process: import("./managed-process.js").ManagedProcessStatus}} ProcessStatus
|
|
26
26
|
* @typedef {{disruptive: true, mode: "legacy-first-upgrade", reason: string}} OwnerTransition
|
|
27
|
-
* @typedef {"candidate_ready" | "retiring_previous" | "previous_retired" | "activating_candidate" | "committed_pending" | "committed" | "restoring_committed"} GenerationTransitionPhase
|
|
28
|
-
* @typedef {{activationLifecycle?: boolean, candidateReleaseId: string, candidateReleasePath: string, candidateRevision: string, configDigest: string, error?: string, journalRevision?: number, phase: GenerationTransitionPhase, previousReleaseId: string | null, startedAt: string, updatedAt: string}} GenerationTransition
|
|
27
|
+
* @typedef {"candidate_ready" | "retiring_previous" | "previous_retired" | "activating_candidate" | "restoring_previous" | "retiring_failed_candidate" | "committed_pending" | "committed" | "restoring_committed"} GenerationTransitionPhase
|
|
28
|
+
* @typedef {{activationError?: string, activationLifecycle?: boolean, candidateReleaseId: string, candidateReleasePath: string, candidateRevision: string, compensationError?: string, configDigest: string, error?: string, journalRevision?: number, phase: GenerationTransitionPhase, previousReleaseId: string | null, startedAt: string, updatedAt: string}} GenerationTransition
|
|
29
29
|
* @typedef {{activeReleaseId: string | null, application: string, bootstrap: BootstrapIdentity | undefined, control: import("./config.js").ControlConfig, daemonPid: number, daemonRuntime: import("./daemon-runtime.js").DaemonRuntimeIdentity | undefined, generationTransition?: GenerationTransition, ownerRecovery: {configDigest: string, ready: boolean} | undefined, ownerTransition?: OwnerTransition, orphans: {id: string, pid: number, releaseId: string | null}[], proxy: {host: string, port: number | undefined, upstreamHost: string}, releaseReferences: {releaseId: string, releasePath: string}[], releases: import("./release-group.js").ReleaseStatus[], services: ProcessStatus[], singletons: ProcessStatus[]}} DaemonStatus
|
|
30
30
|
* @typedef {{configDigest: string, format: number, guardian: {pid?: number, socketPath: string, token: string}, reconnectGraceMs: number}} OwnerRecoveryMetadata
|
|
31
31
|
* @typedef {DaemonStatus & {recovery: OwnerRecoveryMetadata, serviceReleaseIds?: Record<string, string>, singletonReleaseIds?: Record<string, string>}} OwnerRecoverySnapshot
|
|
@@ -1337,6 +1337,15 @@ export default class RollbridgeDaemon {
|
|
|
1337
1337
|
}))
|
|
1338
1338
|
}
|
|
1339
1339
|
|
|
1340
|
+
if (commandName === "recover-generation-transition") {
|
|
1341
|
+
return await this.executeOwnerMutation("recover generation transition", async () => await this.recoverGenerationTransition({
|
|
1342
|
+
previousReleaseId: requiredString(data.previousReleaseId, "previousReleaseId"),
|
|
1343
|
+
releaseId: requiredString(data.releaseId, "releaseId"),
|
|
1344
|
+
releasePath: requiredString(data.releasePath, "releasePath"),
|
|
1345
|
+
revision: requiredString(data.revision, "revision")
|
|
1346
|
+
}))
|
|
1347
|
+
}
|
|
1348
|
+
|
|
1340
1349
|
if (commandName === "status") {
|
|
1341
1350
|
return this.status()
|
|
1342
1351
|
}
|
|
@@ -1627,6 +1636,9 @@ export default class RollbridgeDaemon {
|
|
|
1627
1636
|
const transition = this.generationTransition
|
|
1628
1637
|
|
|
1629
1638
|
if (!transition) throw new Error("No release generation transition to resume")
|
|
1639
|
+
if (transition.phase === "restoring_previous" || transition.phase === "retiring_failed_candidate") {
|
|
1640
|
+
return await this.compensatePreCommitActivationFailure()
|
|
1641
|
+
}
|
|
1630
1642
|
const release = this.releases.get(transition.candidateReleaseId)
|
|
1631
1643
|
const previousRelease = transition.previousReleaseId ? this.releases.get(transition.previousReleaseId) : undefined
|
|
1632
1644
|
|
|
@@ -1668,7 +1680,23 @@ export default class RollbridgeDaemon {
|
|
|
1668
1680
|
|
|
1669
1681
|
await this.failGenerationTransition(failure)
|
|
1670
1682
|
this.logger("release generation activation failed", {error: failure, releaseId: release.releaseId})
|
|
1671
|
-
throw error
|
|
1683
|
+
if (!previousRelease || !activationLifecycle) throw error
|
|
1684
|
+
|
|
1685
|
+
let compensation
|
|
1686
|
+
|
|
1687
|
+
try {
|
|
1688
|
+
compensation = await this.compensatePreCommitActivationFailure()
|
|
1689
|
+
} catch (compensationError) {
|
|
1690
|
+
throw new AggregateError(
|
|
1691
|
+
[error, compensationError],
|
|
1692
|
+
`${failure}; pre-commit compensation failed: ${compensationError instanceof Error ? compensationError.message : String(compensationError)}`,
|
|
1693
|
+
{cause: compensationError}
|
|
1694
|
+
)
|
|
1695
|
+
}
|
|
1696
|
+
const compensatedError = /** @type {Error & {compensation?: Record<string, JsonValue>}} */ (new Error(`${failure}; compensation restored incumbent ${compensation.previousReleaseId} as authoritative and retired failed candidate ${compensation.candidateReleaseId}`, {cause: error}))
|
|
1697
|
+
|
|
1698
|
+
compensatedError.compensation = compensation
|
|
1699
|
+
throw compensatedError
|
|
1672
1700
|
}
|
|
1673
1701
|
|
|
1674
1702
|
// Activation acknowledgement and the logical proxy commit deliberately share one
|
|
@@ -1732,6 +1760,155 @@ export default class RollbridgeDaemon {
|
|
|
1732
1760
|
}
|
|
1733
1761
|
}
|
|
1734
1762
|
|
|
1763
|
+
/**
|
|
1764
|
+
* Restores the authoritative proxy target and retires only the failed candidate.
|
|
1765
|
+
* Every external effect is preceded by a durable phase checkpoint so recovery can
|
|
1766
|
+
* safely replay the paired lifecycle hooks, which are already idempotent contracts.
|
|
1767
|
+
* @returns {Promise<Record<string, JsonValue>>} Structured recovery result.
|
|
1768
|
+
*/
|
|
1769
|
+
async compensatePreCommitActivationFailure() {
|
|
1770
|
+
const transition = this.generationTransition
|
|
1771
|
+
|
|
1772
|
+
if (!transition) throw new Error("No release generation transition to compensate")
|
|
1773
|
+
if (transition.phase !== "activating_candidate" && transition.phase !== "restoring_previous" && transition.phase !== "retiring_failed_candidate") {
|
|
1774
|
+
throw new Error(`Generation transition ${transition.candidateReleaseId} at ${transition.phase} is not safe for pre-commit compensation`)
|
|
1775
|
+
}
|
|
1776
|
+
if (!transition.previousReleaseId) throw new Error("Pre-commit compensation requires a retained previous release")
|
|
1777
|
+
const candidate = this.releases.get(transition.candidateReleaseId)
|
|
1778
|
+
const previous = this.releases.get(transition.previousReleaseId)
|
|
1779
|
+
|
|
1780
|
+
if (!candidate) throw new Error(`Generation transition candidate ${transition.candidateReleaseId} is not retained`)
|
|
1781
|
+
if (!previous) throw new Error(`Generation transition previous release ${transition.previousReleaseId} is not retained`)
|
|
1782
|
+
if (this.activeRelease !== previous) throw new Error(`Previous release ${previous.releaseId} is not the authoritative proxy target`)
|
|
1783
|
+
|
|
1784
|
+
if (transition.phase === "activating_candidate") {
|
|
1785
|
+
if (!transition.error && !transition.activationError) throw new Error("Candidate activation has no recorded failure to compensate")
|
|
1786
|
+
transition.activationError = transition.activationError || transition.error
|
|
1787
|
+
transition.compensationError = undefined
|
|
1788
|
+
await this.updateGenerationTransition("retiring_failed_candidate")
|
|
1789
|
+
}
|
|
1790
|
+
|
|
1791
|
+
if (transition.phase === "retiring_failed_candidate") {
|
|
1792
|
+
try {
|
|
1793
|
+
await candidate.beginRetirement(candidate.config, {retry: true})
|
|
1794
|
+
const candidateServiceIds = [...this.serviceReleaseIds.entries()]
|
|
1795
|
+
.filter(([, releaseId]) => releaseId === candidate.releaseId)
|
|
1796
|
+
.map(([serviceId]) => serviceId)
|
|
1797
|
+
|
|
1798
|
+
await this.stopStartedServices(candidateServiceIds)
|
|
1799
|
+
} catch (error) {
|
|
1800
|
+
const failure = error instanceof Error ? error.message : String(error)
|
|
1801
|
+
|
|
1802
|
+
transition.compensationError = failure
|
|
1803
|
+
await this.failGenerationTransition(`Failed candidate ${candidate.releaseId} retirement failed: ${failure}`)
|
|
1804
|
+
this.logger("release generation compensation candidate retirement failed", {
|
|
1805
|
+
activationError: transition.activationError,
|
|
1806
|
+
error: failure,
|
|
1807
|
+
releaseId: candidate.releaseId
|
|
1808
|
+
})
|
|
1809
|
+
throw new Error(`failed candidate ${candidate.releaseId} retirement failed: ${failure}`, {cause: error})
|
|
1810
|
+
}
|
|
1811
|
+
transition.compensationError = undefined
|
|
1812
|
+
await this.updateGenerationTransition("restoring_previous")
|
|
1813
|
+
this.logger("release generation compensation candidate retired", {releaseId: candidate.releaseId})
|
|
1814
|
+
}
|
|
1815
|
+
|
|
1816
|
+
if (transition.phase === "restoring_previous") {
|
|
1817
|
+
if (candidate.state !== "draining") throw new Error(`Failed candidate ${candidate.releaseId} is not retired before incumbent restoration`)
|
|
1818
|
+
try {
|
|
1819
|
+
await previous.reactivateGeneration()
|
|
1820
|
+
} catch (error) {
|
|
1821
|
+
const failure = error instanceof Error ? error.message : String(error)
|
|
1822
|
+
|
|
1823
|
+
transition.compensationError = failure
|
|
1824
|
+
await this.failGenerationTransition(`Incumbent ${previous.releaseId} restoration failed: ${failure}`)
|
|
1825
|
+
this.logger("release generation compensation restoration failed", {
|
|
1826
|
+
activationError: transition.activationError,
|
|
1827
|
+
error: failure,
|
|
1828
|
+
releaseId: previous.releaseId
|
|
1829
|
+
})
|
|
1830
|
+
throw new Error(`incumbent ${previous.releaseId} restoration failed: ${failure}`, {cause: error})
|
|
1831
|
+
}
|
|
1832
|
+
this.activeRelease = previous
|
|
1833
|
+
transition.compensationError = undefined
|
|
1834
|
+
this.logger("release generation compensation incumbent restored", {releaseId: previous.releaseId})
|
|
1835
|
+
}
|
|
1836
|
+
|
|
1837
|
+
const result = /** @type {Record<string, JsonValue>} */ ({
|
|
1838
|
+
activeReleaseId: previous.releaseId,
|
|
1839
|
+
activationError: transition.activationError,
|
|
1840
|
+
candidateReleaseId: candidate.releaseId,
|
|
1841
|
+
failedCandidateStatus: candidate.state,
|
|
1842
|
+
previousReleaseId: previous.releaseId,
|
|
1843
|
+
recoveryStatus: "recovered"
|
|
1844
|
+
})
|
|
1845
|
+
|
|
1846
|
+
this.config = previous.config
|
|
1847
|
+
this.generationTransition = undefined
|
|
1848
|
+
try {
|
|
1849
|
+
await this.checkpointGenerationTransition()
|
|
1850
|
+
} catch (error) {
|
|
1851
|
+
const failure = error instanceof Error ? error.message : String(error)
|
|
1852
|
+
|
|
1853
|
+
this.generationTransition = transition
|
|
1854
|
+
transition.compensationError = failure
|
|
1855
|
+
transition.error = `Compensation checkpoint clear failed: ${failure}`
|
|
1856
|
+
transition.journalRevision = (transition.journalRevision ?? 0) + 1
|
|
1857
|
+
transition.updatedAt = new Date().toISOString()
|
|
1858
|
+
await this.publishOwnerState().catch((publishError) => {
|
|
1859
|
+
this.logger("release generation compensation fence republish failed", {
|
|
1860
|
+
error: publishError instanceof Error ? publishError.message : String(publishError),
|
|
1861
|
+
releaseId: candidate.releaseId
|
|
1862
|
+
})
|
|
1863
|
+
})
|
|
1864
|
+
this.logger("release generation compensation checkpoint failed", {error: failure, releaseId: candidate.releaseId})
|
|
1865
|
+
throw new Error(`compensation checkpoint clear failed: ${failure}`, {cause: error})
|
|
1866
|
+
}
|
|
1867
|
+
void this.drainAndPrune(candidate, candidate.config)
|
|
1868
|
+
this.logger("release generation compensation completed", result)
|
|
1869
|
+
return result
|
|
1870
|
+
}
|
|
1871
|
+
|
|
1872
|
+
/**
|
|
1873
|
+
* Authenticated control recovery for one exact failed pre-commit transition.
|
|
1874
|
+
* @param {{previousReleaseId: string, releaseId: string, releasePath: string, revision: string}} identity - Exact transition fence.
|
|
1875
|
+
* @returns {Promise<Record<string, JsonValue>>} Structured recovery result.
|
|
1876
|
+
*/
|
|
1877
|
+
async recoverGenerationTransition(identity) {
|
|
1878
|
+
const transition = this.generationTransition
|
|
1879
|
+
|
|
1880
|
+
if (!transition) {
|
|
1881
|
+
if (this.activeRelease?.releaseId !== identity.previousReleaseId || this.activeRelease.releaseId === identity.releaseId) {
|
|
1882
|
+
throw new Error("No matching recovered generation transition; the expected incumbent is not authoritative")
|
|
1883
|
+
}
|
|
1884
|
+
return {
|
|
1885
|
+
activeReleaseId: this.activeRelease.releaseId,
|
|
1886
|
+
candidateReleaseId: identity.releaseId,
|
|
1887
|
+
previousReleaseId: identity.previousReleaseId,
|
|
1888
|
+
recoveryStatus: "already_recovered"
|
|
1889
|
+
}
|
|
1890
|
+
}
|
|
1891
|
+
if (transition.previousReleaseId !== identity.previousReleaseId) {
|
|
1892
|
+
throw new Error(`Generation transition previous release is ${transition.previousReleaseId}; refusing stale recovery for ${identity.previousReleaseId}`)
|
|
1893
|
+
}
|
|
1894
|
+
if (transition.phase !== "activating_candidate" && transition.phase !== "restoring_previous" && transition.phase !== "retiring_failed_candidate") {
|
|
1895
|
+
throw new Error(`Generation transition ${transition.candidateReleaseId} at ${transition.phase} is not a safe failed pre-commit transition`)
|
|
1896
|
+
}
|
|
1897
|
+
const nextConfig = this.configPath ? await loadConfig(this.configPath) : this.config
|
|
1898
|
+
|
|
1899
|
+
this.assertReloadCompatible(nextConfig)
|
|
1900
|
+
this.assertExactGenerationTransition(transition, {
|
|
1901
|
+
config: nextConfig,
|
|
1902
|
+
releaseId: identity.releaseId,
|
|
1903
|
+
releasePath: identity.releasePath,
|
|
1904
|
+
revision: identity.revision
|
|
1905
|
+
})
|
|
1906
|
+
if (this.activeRelease?.releaseId !== transition.previousReleaseId) {
|
|
1907
|
+
throw new Error(`Previous release ${transition.previousReleaseId} is not the authoritative proxy target`)
|
|
1908
|
+
}
|
|
1909
|
+
return await this.compensatePreCommitActivationFailure()
|
|
1910
|
+
}
|
|
1911
|
+
|
|
1735
1912
|
/**
|
|
1736
1913
|
* @param {GenerationTransition} transition - Pending or committed exact transition.
|
|
1737
1914
|
* @param {{config: import("./config.js").RollbridgeConfig, releaseId: string, releasePath: string, revision: string}} candidate - Requested identity.
|
package/src/guardian-client.js
CHANGED
|
@@ -23,6 +23,7 @@ export default class GuardianClient {
|
|
|
23
23
|
this.processes = /** @type {Map<string, GuardianProcess>} */ (new Map())
|
|
24
24
|
this.reservedProcessKey = /** @type {string | undefined} */ (undefined)
|
|
25
25
|
this.reservedProcessProvenance = /** @type {string | undefined} */ (undefined)
|
|
26
|
+
this.generationReactivation = /** @type {boolean | undefined} */ (undefined)
|
|
26
27
|
this.events = /** @type {Map<string, {reject: (error: Error) => void, resolve: (value: Record<string, import("./json.js").JsonValue>) => void}[]>} */ (new Map())
|
|
27
28
|
this.eventHandlers = /** @type {Map<string, ((event: Record<string, import("./json.js").JsonValue>) => void)[]>} */ (new Map())
|
|
28
29
|
}
|
|
@@ -51,6 +52,7 @@ export default class GuardianClient {
|
|
|
51
52
|
if (child.connected) await new Promise((resolve) => child.once("disconnect", () => resolve(undefined)))
|
|
52
53
|
child.unref()
|
|
53
54
|
await this.connect()
|
|
55
|
+
this.generationReactivation = true
|
|
54
56
|
}
|
|
55
57
|
|
|
56
58
|
/**
|
|
@@ -184,9 +186,12 @@ export default class GuardianClient {
|
|
|
184
186
|
await this.request({command: "owner-ready", ownerPid: process.pid})
|
|
185
187
|
}
|
|
186
188
|
|
|
187
|
-
/** @returns {Promise<{daemonRecovery: number}>} Guardian protocol capabilities. */
|
|
189
|
+
/** @returns {Promise<{daemonRecovery: number, generationReactivation?: number}>} Guardian protocol capabilities. */
|
|
188
190
|
async capabilities() {
|
|
189
|
-
|
|
191
|
+
const capabilities = /** @type {{daemonRecovery: number, generationReactivation?: number}} */ (await this.request({command: "capabilities"}))
|
|
192
|
+
|
|
193
|
+
this.generationReactivation = capabilities.generationReactivation === 1
|
|
194
|
+
return capabilities
|
|
190
195
|
}
|
|
191
196
|
|
|
192
197
|
/** Starts graceful process retirement and relinquishes committed owner authority. */
|
|
@@ -445,6 +450,7 @@ class GuardianProcess extends ManagedProcess {
|
|
|
445
450
|
this.cachedStatus = super.status()
|
|
446
451
|
this.registration = /** @type {Promise<void> | undefined} */ (undefined)
|
|
447
452
|
this.pendingUpdate = Promise.resolve()
|
|
453
|
+
this.compatibilityReactivated = false
|
|
448
454
|
}
|
|
449
455
|
|
|
450
456
|
async ensureRegistered() {
|
|
@@ -479,6 +485,7 @@ class GuardianProcess extends ManagedProcess {
|
|
|
479
485
|
await this.pendingUpdate
|
|
480
486
|
if (lifecycleRole) this.lifecycleRole = lifecycleRole
|
|
481
487
|
this.cachedStatus = asProcessStatus(await this.client.request({command: "start", key: this.key, lifecycleRole, reason}))
|
|
488
|
+
if (this.cachedStatus.state === "running") this.compatibilityReactivated = false
|
|
482
489
|
}
|
|
483
490
|
|
|
484
491
|
/**
|
|
@@ -529,12 +536,52 @@ class GuardianProcess extends ManagedProcess {
|
|
|
529
536
|
this.lifecycleRole = "active"
|
|
530
537
|
}
|
|
531
538
|
|
|
539
|
+
async reactivateStrict() {
|
|
540
|
+
await this.ensureRegistered()
|
|
541
|
+
await this.pendingUpdate
|
|
542
|
+
const command = this.lifecycle.reactivateCommand ? "reactivate-with-command" : "reactivate"
|
|
543
|
+
try {
|
|
544
|
+
this.cachedStatus = asProcessStatus(await this.client.request({command, key: this.key}))
|
|
545
|
+
this.client.generationReactivation = true
|
|
546
|
+
this.compatibilityReactivated = false
|
|
547
|
+
} catch (error) {
|
|
548
|
+
if (!(error instanceof Error) || error.message !== `Unknown guardian command: ${command}`) throw error
|
|
549
|
+
this.client.generationReactivation = false
|
|
550
|
+
await this.adoptRetainedActive({activate: true})
|
|
551
|
+
}
|
|
552
|
+
this.lifecycleRole = "active"
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
/**
|
|
556
|
+
* Bridges a retained quiesced process owned by a pre-reactivation guardian.
|
|
557
|
+
* @param {{activate: boolean}} options - Whether to run the external activation acknowledgement.
|
|
558
|
+
*/
|
|
559
|
+
async adoptRetainedActive({activate}) {
|
|
560
|
+
const before = this.cachedStatus
|
|
561
|
+
|
|
562
|
+
if (!before.pid || (before.state !== "quiesced" && before.state !== "running")) throw new Error(`Process ${this.id} is not retained for compatible reactivation`)
|
|
563
|
+
if (activate) await this.runReactivationHook(before.pid)
|
|
564
|
+
const verified = asProcessStatus(await this.client.request({command: "status", key: this.key}))
|
|
565
|
+
|
|
566
|
+
if (verified.pid !== before.pid || (verified.state !== "quiesced" && verified.state !== "running")) {
|
|
567
|
+
throw new Error(`Process ${this.id} exited before compatible reactivation completed`)
|
|
568
|
+
}
|
|
569
|
+
const adopted = asProcessStatus(await this.client.request({command: "start", key: this.key, lifecycleRole: "active", reason: "deploy"}))
|
|
570
|
+
|
|
571
|
+
if (adopted.pid !== before.pid) throw new Error(`Process ${this.id} changed before compatible reactivation was adopted`)
|
|
572
|
+
this.cachedStatus = asProcessStatus({...adopted, lifecycleRole: "active", state: "running"})
|
|
573
|
+
this.compatibilityReactivated = true
|
|
574
|
+
}
|
|
575
|
+
|
|
532
576
|
/** @param {import("./managed-process.js").LifecycleRole} role - Exact generation role. */
|
|
533
577
|
async setLifecycleRole(role) {
|
|
534
578
|
await this.ensureRegistered()
|
|
535
579
|
await this.pendingUpdate
|
|
536
580
|
this.cachedStatus = asProcessStatus(await this.client.request({command: "set-lifecycle-role", key: this.key, lifecycleRole: role}))
|
|
537
581
|
this.lifecycleRole = role
|
|
582
|
+
if (role === "active" && this.client.generationReactivation === false && this.cachedStatus.state === "quiesced") {
|
|
583
|
+
await this.adoptRetainedActive({activate: false})
|
|
584
|
+
}
|
|
538
585
|
}
|
|
539
586
|
|
|
540
587
|
async stop(options = {}) {
|
|
@@ -550,6 +597,12 @@ class GuardianProcess extends ManagedProcess {
|
|
|
550
597
|
/** @param {Record<string, import("./json.js").JsonValue>} event - Guardian event. */
|
|
551
598
|
onGuardianEvent(event) {
|
|
552
599
|
if (event.status) this.cachedStatus = asProcessStatus(event.status)
|
|
600
|
+
if (this.compatibilityReactivated && (this.cachedStatus.state === "failed" || this.cachedStatus.state === "stopped") && this.shouldRestart()) {
|
|
601
|
+
this.compatibilityReactivated = false
|
|
602
|
+
void this.start("crash", this.lifecycleRole).catch((error) => {
|
|
603
|
+
this.logger("compatible reactivated process restart failed", {error: error instanceof Error ? error.message : String(error), id: this.id})
|
|
604
|
+
})
|
|
605
|
+
}
|
|
553
606
|
if (event.event === "process-log") {
|
|
554
607
|
const entry = asProcessLog(event.entry)
|
|
555
608
|
|
package/src/managed-process.js
CHANGED
|
@@ -4,7 +4,7 @@ import {EventEmitter} from "node:events"
|
|
|
4
4
|
import {spawn} from "node:child_process"
|
|
5
5
|
import {processGroupHasLiveMembers, processGroupMembers} from "./process-memory.js"
|
|
6
6
|
|
|
7
|
-
const
|
|
7
|
+
const DEFAULT_ACTIVATION_HOOK_TIMEOUT_MS = 30000
|
|
8
8
|
const MAX_BUFFERED_OUTPUT_CHARACTERS = 64 * 1024
|
|
9
9
|
|
|
10
10
|
/**
|
|
@@ -486,14 +486,14 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
486
486
|
if (this.lifecycleRestoreBarrier) await this.lifecycleRestoreBarrier
|
|
487
487
|
if (!this.child?.pid) {
|
|
488
488
|
this.state = "stopped"
|
|
489
|
-
if (this.
|
|
489
|
+
if (this.hasRestorableLifecycle()) this.lifecycleRole = "retired"
|
|
490
490
|
return
|
|
491
491
|
}
|
|
492
492
|
this.state = "stopping"
|
|
493
493
|
if (this.lifecycle.quietCommand) this.quiesceError = await this.runHook(this.lifecycle.quietCommand, this.hookTimeoutMs(), "quiet command")
|
|
494
494
|
if (!this.quiesceError) {
|
|
495
495
|
this.state = "quiesced"
|
|
496
|
-
if (this.
|
|
496
|
+
if (this.hasRestorableLifecycle()) this.lifecycleRole = "retired"
|
|
497
497
|
}
|
|
498
498
|
})()
|
|
499
499
|
return await this.quiescePromise
|
|
@@ -521,13 +521,65 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
521
521
|
const pid = this.pid
|
|
522
522
|
|
|
523
523
|
if (!child?.pid || child.pid !== pid || this.state !== "running") throw new Error(`Process ${this.id} is not running for activation`)
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
if (error) throw error
|
|
524
|
+
await this.runActivationHook(pid)
|
|
527
525
|
if (this.child !== child || this.pid !== pid || this.state !== "running") throw new Error(`Process ${this.id} exited before activation completed`)
|
|
528
526
|
this.lifecycleRole = "active"
|
|
529
527
|
}
|
|
530
528
|
|
|
529
|
+
/** Reactivates an explicitly retired process without replacing its release-scoped process. */
|
|
530
|
+
async reactivateStrict() {
|
|
531
|
+
const child = this.child
|
|
532
|
+
const pid = this.pid
|
|
533
|
+
|
|
534
|
+
if (!child?.pid || child.pid !== pid || (this.state !== "quiesced" && this.state !== "running")) {
|
|
535
|
+
throw new Error(`Process ${this.id} is not retained for reactivation`)
|
|
536
|
+
}
|
|
537
|
+
await this.runReactivationHook(pid)
|
|
538
|
+
if (this.child !== child || this.pid !== pid) throw new Error(`Process ${this.id} exited before reactivation completed`)
|
|
539
|
+
this.intentionalStop = false
|
|
540
|
+
this.intentionalStopSignal = undefined
|
|
541
|
+
this.quiescePromise = undefined
|
|
542
|
+
this.quiesceError = undefined
|
|
543
|
+
this.state = "running"
|
|
544
|
+
this.lifecycleRole = "active"
|
|
545
|
+
this.startMemoryMonitor()
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
/**
|
|
549
|
+
* Runs the generation activation hook against one exact retained process.
|
|
550
|
+
* @param {number | undefined} pid - Exact process group leader.
|
|
551
|
+
* @returns {Promise<void>} Resolves after acknowledgement.
|
|
552
|
+
*/
|
|
553
|
+
async runActivationHook(pid) {
|
|
554
|
+
const command = this.lifecycle.activateCommand
|
|
555
|
+
|
|
556
|
+
if (!command) return
|
|
557
|
+
const error = await this.runHook(command, this.activationHookTimeoutMs(), "activate command", pid)
|
|
558
|
+
|
|
559
|
+
if (error) throw error
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
/**
|
|
563
|
+
* Runs the process-specific reactivation hook, falling back to the generation
|
|
564
|
+
* coordinator's activation hook for the original lifecycle contract.
|
|
565
|
+
* @param {number | undefined} pid - Exact process group leader.
|
|
566
|
+
* @returns {Promise<void>} Resolves after acknowledgement.
|
|
567
|
+
*/
|
|
568
|
+
async runReactivationHook(pid) {
|
|
569
|
+
const command = this.lifecycle.reactivateCommand ?? this.lifecycle.activateCommand
|
|
570
|
+
|
|
571
|
+
if (!command) return
|
|
572
|
+
const label = this.lifecycle.reactivateCommand ? "reactivate command" : "activate command"
|
|
573
|
+
const error = await this.runHook(command, this.activationHookTimeoutMs(), label, pid)
|
|
574
|
+
|
|
575
|
+
if (error) throw error
|
|
576
|
+
}
|
|
577
|
+
|
|
578
|
+
/** @returns {boolean} Whether this process owns a durable external lifecycle role. */
|
|
579
|
+
hasRestorableLifecycle() {
|
|
580
|
+
return Boolean(this.lifecycle.activateCommand || this.lifecycle.reactivateCommand)
|
|
581
|
+
}
|
|
582
|
+
|
|
531
583
|
/**
|
|
532
584
|
* Records the durable desired role without firing a lifecycle command.
|
|
533
585
|
* @param {LifecycleRole} role - Exact role owned by this process generation.
|
|
@@ -538,16 +590,24 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
538
590
|
|
|
539
591
|
/** Restores an active or retired role after this exact process starts. */
|
|
540
592
|
async restoreLifecycleRole() {
|
|
541
|
-
if (!this.
|
|
542
|
-
const command = this.lifecycleRole === "active"
|
|
593
|
+
if (!this.hasRestorableLifecycle() || this.lifecycleRole === "candidate") return
|
|
594
|
+
const command = this.lifecycleRole === "active"
|
|
595
|
+
? this.lifecycle.reactivateCommand ?? this.lifecycle.activateCommand
|
|
596
|
+
: this.lifecycle.quietCommand
|
|
543
597
|
|
|
544
598
|
if (!command) throw new Error(`Process ${this.id} cannot restore lifecycle role ${this.lifecycleRole} without its paired command`)
|
|
545
|
-
const timeoutMs = this.lifecycleRole === "active" ?
|
|
546
|
-
const
|
|
599
|
+
const timeoutMs = this.lifecycleRole === "active" ? this.activationHookTimeoutMs() : this.hookTimeoutMs()
|
|
600
|
+
const activeLabel = this.lifecycle.reactivateCommand ? "reactivate" : "activate"
|
|
601
|
+
const error = await this.runHook(command, timeoutMs, `${this.lifecycleRole === "active" ? activeLabel : "quiet"} command`, this.pid)
|
|
547
602
|
|
|
548
603
|
if (error) throw error
|
|
549
604
|
}
|
|
550
605
|
|
|
606
|
+
/** @returns {number} Timeout used for activation/reactivation hooks. */
|
|
607
|
+
activationHookTimeoutMs() {
|
|
608
|
+
return this.lifecycle.activateTimeoutMs ?? DEFAULT_ACTIVATION_HOOK_TIMEOUT_MS
|
|
609
|
+
}
|
|
610
|
+
|
|
551
611
|
/** @returns {number} Timeout used for lifecycle hooks. */
|
|
552
612
|
hookTimeoutMs() {
|
|
553
613
|
if (this.stopTimeoutMs === "indefinite") return 30000
|
|
@@ -760,7 +820,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
760
820
|
id: this.id,
|
|
761
821
|
lastMemoryRestartAt: this.lastMemoryRestartAtMs === undefined ? undefined : new Date(this.lastMemoryRestartAtMs).toISOString(),
|
|
762
822
|
lastStartReason: this.lastStartReason,
|
|
763
|
-
...(this.
|
|
823
|
+
...(this.hasRestorableLifecycle() ? {lifecycleRole: this.lifecycleRole} : {}),
|
|
764
824
|
logs: this.logs.slice(-this.outputLines),
|
|
765
825
|
memoryRestarts: this.memoryRestarts,
|
|
766
826
|
pid: this.pid,
|