rollbridge 0.1.40 → 0.1.42

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -89,6 +89,7 @@ export default {
89
89
  command: "env VELOCIOUS_BACKGROUND_JOBS_PORT={{port}} npx velocious background-jobs-main",
90
90
  lifecycle: {
91
91
  activateCommand: 'npx velocious background-jobs:activate --generation "$ROLLBRIDGE_RELEASE_ID" --socket "$VELOCIOUS_BACKGROUND_JOBS_LIFECYCLE_SOCKET"',
92
+ activateTimeoutMs: 60000,
92
93
  quietCommand: 'npx velocious background-jobs:retire --generation "$ROLLBRIDGE_RELEASE_ID" --socket "$VELOCIOUS_BACKGROUND_JOBS_LIFECYCLE_SOCKET"'
93
94
  },
94
95
  port: {from: 7331, to: 7399}
@@ -188,7 +189,8 @@ generation-scoped and resumable; failures remain visible and block unrelated
188
189
  deploys. Post-commit singleton replacement is also journaled and must complete
189
190
  before an exact retry reports success. If the active coordinator restarts,
190
191
  Rollbridge restores its active role with the same bounded, generation-scoped
191
- activation command before reporting it running. Omit `activateCommand` to
192
+ activation command before reporting it running. Set `activateTimeoutMs` when the
193
+ activation acknowledgement can exceed its 30-second default. Omit `activateCommand` to
192
194
  preserve the existing hook-free ordering.
193
195
 
194
196
  See [`docs/workers.md`](docs/workers.md) for the full release-generation
package/docs/config.md CHANGED
@@ -303,8 +303,8 @@ service when the service starts as a quiescent candidate and requires an explici
303
303
  generation transition. Rollbridge starts and health-checks the complete candidate,
304
304
  waits for the old generation's strict retirement acknowledgement, waits for the
305
305
  candidate's strict activation acknowledgement, then commits the active release and
306
- proxy target synchronously. Activation is always bounded to 30 seconds;
307
- retirement uses the process's `gracefulStopMs` bound (or 30 seconds when that
306
+ proxy target synchronously. Activation is bounded by `lifecycle.activateTimeoutMs`,
307
+ which defaults to 30 seconds; retirement uses the process's `gracefulStopMs` bound (or 30 seconds when that
308
308
  window is `"indefinite"`). Both run with the process environment plus
309
309
  `ROLLBRIDGE_PID`.
310
310
 
@@ -347,7 +347,8 @@ legitimate hours-long generation drains are valid.
347
347
 
348
348
  | Field | Type | Default | Description |
349
349
  | --- | --- | --- | --- |
350
- | `lifecycle.activateCommand` | string | unset | For one handoff service, acknowledge activation of its already-started candidate generation after the previous generation has acknowledged retirement. Requires `quietCommand`, `statePath`, and `ownerRecovery`; bounded to 30 seconds. |
350
+ | `lifecycle.activateCommand` | string | unset | For one handoff service, acknowledge activation of its already-started candidate generation after the previous generation has acknowledged retirement. Requires `quietCommand`, `statePath`, and `ownerRecovery`; bounded by `activateTimeoutMs`. |
351
+ | `lifecycle.activateTimeoutMs` | positive number | `30000` | Bounds activation, reactivation, and active-role restoration commands. |
351
352
  | `lifecycle.quietCommand` | string | unset | Run first to tell the process to stop accepting new work. Bounded by `gracefulStopMs`, or 30 seconds when that window is `"indefinite"`. |
352
353
  | `lifecycle.drainCommand` | string | unset | Run after quieting to wait until the process has drained (it blocks until done). When unset, Rollbridge instead waits up to `drainTimeoutMs` for the process to exit on its own. Requires a positive `drainTimeoutMs` (which bounds it). |
353
354
  | `lifecycle.drainTimeoutMs` | non-negative number | `0` | Bounds the drain step. `0` **skips the drain step entirely** (no `drainCommand`, no wait). |
@@ -4,7 +4,7 @@
4
4
  // binds that stable HTTP port, forwards to the active release's internal web
5
5
  // port and keeps Beacon daemon-wide. The lifecycle socket path must match the
6
6
  // reviewed release-local Velocious jobs-main configuration; the worker appctl
7
- // command remains an illustrative application-specific quiescence control.
7
+ // commands remain illustrative application-specific quiescence/reactivation controls.
8
8
 
9
9
  export default {
10
10
  application: "tensorbuzz",
@@ -68,7 +68,10 @@ export default {
68
68
  VELOCIOUS_BACKGROUND_JOBS_PORT: "{{ports.background-jobs-main}}"
69
69
  },
70
70
  command: "wait-for-it 127.0.0.1:{{ports.beacon}} --strict -- wait-for-it 127.0.0.1:{{ports.background-jobs-main}} --strict -- npx velocious background-jobs-worker",
71
- lifecycle: {quietCommand: "appctl jobs-worker-retire --pid $ROLLBRIDGE_PID"},
71
+ lifecycle: {
72
+ quietCommand: "appctl jobs-worker-retire --pid $ROLLBRIDGE_PID",
73
+ reactivateCommand: "appctl jobs-worker-reactivate --pid $ROLLBRIDGE_PID"
74
+ },
72
75
  nonBlockingDrain: true,
73
76
  gracefulStopMs: "indefinite"
74
77
  },
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "rollbridge",
3
- "version": "0.1.40",
3
+ "version": "0.1.42",
4
4
  "description": "Zero-downtime process supervisor and local traffic switcher for deploy-managed apps.",
5
5
  "keywords": [
6
6
  "deploy",
package/src/cli.js CHANGED
@@ -213,6 +213,31 @@ export async function runCli(argv) {
213
213
  console.log(JSON.stringify(response, null, 2))
214
214
  })
215
215
 
216
+ program
217
+ .command("recover-generation-transition")
218
+ .description("Restore the exact incumbent and retire a failed pre-commit generation candidate.")
219
+ .option("-c, --config <path>", "Config file path (defaults to rollbridge.js)")
220
+ .requiredOption("--release-path <path>", "Failed candidate release path")
221
+ .requiredOption("--release-id <id>", "Failed candidate release id")
222
+ .requiredOption("--revision <sha>", "Failed candidate revision")
223
+ .requiredOption("--previous-release-id <id>", "Expected authoritative incumbent release id")
224
+ .action(async (options) => {
225
+ const configPath = await resolveConfigPath(options.config)
226
+ const config = await loadConfig(configPath)
227
+ const response = await sendControlCommand({
228
+ command: {
229
+ command: "recover-generation-transition",
230
+ previousReleaseId: options.previousReleaseId,
231
+ releaseId: options.releaseId,
232
+ releasePath: options.releasePath,
233
+ revision: options.revision
234
+ },
235
+ path: config.control.path
236
+ })
237
+
238
+ console.log(JSON.stringify(response, null, 2))
239
+ })
240
+
216
241
  program
217
242
  .command("ensure-daemon")
218
243
  .description("Start the daemon if the control socket is not already accepting commands.")
package/src/config.js CHANGED
@@ -14,7 +14,7 @@ import {pathToFileURL} from "node:url"
14
14
  * @typedef {"proxied" | "companion" | "singleton" | "service"} ProcessPolicy
15
15
  * @typedef {{backoffFactor: number, maxDelayMs: number, maxRestarts: number | undefined, windowMs: number}} RestartConfig
16
16
  * @typedef {{checkIntervalMs: number, limitBytes: number, warnBytes: number}} MemoryConfig
17
- * @typedef {{activateCommand?: string, drainCommand?: string, drainTimeoutMs: number, quietCommand?: string, stopCommand?: string}} LifecycleConfig
17
+ * @typedef {{activateCommand?: string, activateTimeoutMs?: number, drainCommand?: string, drainTimeoutMs: number, quietCommand?: string, reactivateCommand?: string, stopCommand?: string}} LifecycleConfig
18
18
  * @typedef {number | "indefinite"} StopTimeoutMs
19
19
  * @typedef {"persistent" | "handoff"} ServiceDeployStrategy
20
20
  * @typedef {{cwd?: string, deployStrategy: ServiceDeployStrategy, env: Record<string, string>, gracefulStopMs: StopTimeoutMs, health?: HealthConfig, id: string, lifecycle: LifecycleConfig, memory?: MemoryConfig, nonBlockingDrain: boolean, outputLines: number, policy: ProcessPolicy, port?: PortRange, replicas: number, restart: RestartConfig, restartDelayMs: number, stopSignal: string, command: string}} ProcessConfig
@@ -343,20 +343,29 @@ function normalizeMemory(value, key, issues) {
343
343
  * @returns {LifecycleConfig} Normalized lifecycle hooks (no commands and a 0 drain by default).
344
344
  */
345
345
  function normalizeLifecycle(value, key, issues) {
346
- if (value === undefined || value === null) return {drainTimeoutMs: 0}
346
+ if (value === undefined || value === null) return {activateTimeoutMs: 30000, drainTimeoutMs: 0}
347
347
 
348
348
  if (!isPlainObject(value)) {
349
- issues.push({fix: `Set ${key} to a mapping with optional activateCommand, quietCommand, drainCommand, stopCommand, and drainTimeoutMs.`, message: `${key} must be an object`})
349
+ issues.push({fix: `Set ${key} to a mapping with optional activateCommand, quietCommand, reactivateCommand, drainCommand, stopCommand, and drainTimeoutMs.`, message: `${key} must be an object`})
350
350
 
351
- return {drainTimeoutMs: 0}
351
+ return {activateTimeoutMs: 30000, drainTimeoutMs: 0}
352
352
  }
353
353
 
354
+ const activateTimeoutMs = normalizeNumber(value.activateTimeoutMs, `${key}.activateTimeoutMs`, issues, {default: 30000})
354
355
  const drainTimeoutMs = normalizeNumber(value.drainTimeoutMs, `${key}.drainTimeoutMs`, issues, {default: 0})
355
356
  /** @type {LifecycleConfig} */
356
- const lifecycle = {drainTimeoutMs: nonNegativeOrDefault(drainTimeoutMs, `${key}.drainTimeoutMs`, issues, 0, false)}
357
+ const lifecycle = {
358
+ activateTimeoutMs: activateTimeoutMs > 0 ? activateTimeoutMs : 30000,
359
+ drainTimeoutMs: nonNegativeOrDefault(drainTimeoutMs, `${key}.drainTimeoutMs`, issues, 0, false)
360
+ }
361
+
362
+ if (activateTimeoutMs <= 0) {
363
+ issues.push({fix: `Set ${key}.activateTimeoutMs to a positive number of milliseconds, e.g. 30000.`, message: `${key}.activateTimeoutMs must be a positive number`})
364
+ }
357
365
 
358
366
  if (value.activateCommand !== undefined) lifecycle.activateCommand = normalizeString(value.activateCommand, `${key}.activateCommand`, issues, {nonEmpty: true})
359
367
  if (value.quietCommand !== undefined) lifecycle.quietCommand = normalizeString(value.quietCommand, `${key}.quietCommand`, issues, {nonEmpty: true})
368
+ if (value.reactivateCommand !== undefined) lifecycle.reactivateCommand = normalizeString(value.reactivateCommand, `${key}.reactivateCommand`, issues, {nonEmpty: true})
360
369
  if (value.drainCommand !== undefined) lifecycle.drainCommand = normalizeString(value.drainCommand, `${key}.drainCommand`, issues, {nonEmpty: true})
361
370
  if (value.stopCommand !== undefined) lifecycle.stopCommand = normalizeString(value.stopCommand, `${key}.stopCommand`, issues, {nonEmpty: true})
362
371
 
@@ -376,6 +385,7 @@ function normalizeLifecycle(value, key, issues) {
376
385
  */
377
386
  function validateActivationLifecycle(processes, ownerRecovery, statePath, issues) {
378
387
  const activated = processes.filter((processConfig) => processConfig.lifecycle.activateCommand !== undefined)
388
+ const reactivated = processes.filter((processConfig) => processConfig.lifecycle.reactivateCommand !== undefined)
379
389
 
380
390
  if (activated.length > 1) {
381
391
  issues.push({fix: "Configure lifecycle.activateCommand on only one release-generation coordinator.", message: "Config may define at most one lifecycle.activateCommand"})
@@ -390,6 +400,23 @@ function validateActivationLifecycle(processes, ownerRecovery, statePath, issues
390
400
  }
391
401
  }
392
402
 
403
+ for (const processConfig of reactivated) {
404
+ if (processConfig.policy !== "companion" || !processConfig.nonBlockingDrain) {
405
+ issues.push({fix: `Set lifecycle.reactivateCommand only on a nonBlockingDrain companion; "${processConfig.id}" is ${processConfig.policy} with nonBlockingDrain: ${processConfig.nonBlockingDrain}.`, message: `Process "${processConfig.id}" can only set lifecycle.reactivateCommand on a nonBlockingDrain companion`})
406
+ }
407
+ if (!processConfig.lifecycle.quietCommand) {
408
+ issues.push({fix: `Add lifecycle.quietCommand to "${processConfig.id}" so worker reactivation has a paired retirement command.`, message: `Process "${processConfig.id}" lifecycle.reactivateCommand requires lifecycle.quietCommand`})
409
+ }
410
+ }
411
+
412
+ if (activated.length > 0) {
413
+ for (const processConfig of processes) {
414
+ if (processConfig.nonBlockingDrain && processConfig.lifecycle.quietCommand && !processConfig.lifecycle.reactivateCommand) {
415
+ issues.push({fix: `Add lifecycle.reactivateCommand to "${processConfig.id}" so failed candidate recovery reverses its external quiet hook.`, message: `Process "${processConfig.id}" lifecycle.quietCommand requires lifecycle.reactivateCommand during generation activation recovery`})
416
+ }
417
+ }
418
+ }
419
+
393
420
  if (activated.length > 0 && (!ownerRecovery || !statePath)) {
394
421
  issues.push({fix: "Configure statePath and ownerRecovery for durable release-generation transition recovery.", message: "lifecycle.activateCommand requires ownerRecovery and statePath"})
395
422
  }
package/src/daemon.js CHANGED
@@ -24,8 +24,8 @@ const STATE_PERSIST_INTERVAL_MS = 5000
24
24
  * @typedef {{attestation?: string, releaseId: string, releasePath: string, revision: string}} BootstrapIdentity
25
25
  * @typedef {{id: string, process: import("./managed-process.js").ManagedProcessStatus}} ProcessStatus
26
26
  * @typedef {{disruptive: true, mode: "legacy-first-upgrade", reason: string}} OwnerTransition
27
- * @typedef {"candidate_ready" | "retiring_previous" | "previous_retired" | "activating_candidate" | "committed_pending" | "committed" | "restoring_committed"} GenerationTransitionPhase
28
- * @typedef {{activationLifecycle?: boolean, candidateReleaseId: string, candidateReleasePath: string, candidateRevision: string, configDigest: string, error?: string, journalRevision?: number, phase: GenerationTransitionPhase, previousReleaseId: string | null, startedAt: string, updatedAt: string}} GenerationTransition
27
+ * @typedef {"candidate_ready" | "retiring_previous" | "previous_retired" | "activating_candidate" | "restoring_previous" | "retiring_failed_candidate" | "committed_pending" | "committed" | "restoring_committed"} GenerationTransitionPhase
28
+ * @typedef {{activationError?: string, activationLifecycle?: boolean, candidateReleaseId: string, candidateReleasePath: string, candidateRevision: string, compensationError?: string, configDigest: string, error?: string, journalRevision?: number, phase: GenerationTransitionPhase, previousReleaseId: string | null, startedAt: string, updatedAt: string}} GenerationTransition
29
29
  * @typedef {{activeReleaseId: string | null, application: string, bootstrap: BootstrapIdentity | undefined, control: import("./config.js").ControlConfig, daemonPid: number, daemonRuntime: import("./daemon-runtime.js").DaemonRuntimeIdentity | undefined, generationTransition?: GenerationTransition, ownerRecovery: {configDigest: string, ready: boolean} | undefined, ownerTransition?: OwnerTransition, orphans: {id: string, pid: number, releaseId: string | null}[], proxy: {host: string, port: number | undefined, upstreamHost: string}, releaseReferences: {releaseId: string, releasePath: string}[], releases: import("./release-group.js").ReleaseStatus[], services: ProcessStatus[], singletons: ProcessStatus[]}} DaemonStatus
30
30
  * @typedef {{configDigest: string, format: number, guardian: {pid?: number, socketPath: string, token: string}, reconnectGraceMs: number}} OwnerRecoveryMetadata
31
31
  * @typedef {DaemonStatus & {recovery: OwnerRecoveryMetadata, serviceReleaseIds?: Record<string, string>, singletonReleaseIds?: Record<string, string>}} OwnerRecoverySnapshot
@@ -1337,6 +1337,15 @@ export default class RollbridgeDaemon {
1337
1337
  }))
1338
1338
  }
1339
1339
 
1340
+ if (commandName === "recover-generation-transition") {
1341
+ return await this.executeOwnerMutation("recover generation transition", async () => await this.recoverGenerationTransition({
1342
+ previousReleaseId: requiredString(data.previousReleaseId, "previousReleaseId"),
1343
+ releaseId: requiredString(data.releaseId, "releaseId"),
1344
+ releasePath: requiredString(data.releasePath, "releasePath"),
1345
+ revision: requiredString(data.revision, "revision")
1346
+ }))
1347
+ }
1348
+
1340
1349
  if (commandName === "status") {
1341
1350
  return this.status()
1342
1351
  }
@@ -1627,6 +1636,9 @@ export default class RollbridgeDaemon {
1627
1636
  const transition = this.generationTransition
1628
1637
 
1629
1638
  if (!transition) throw new Error("No release generation transition to resume")
1639
+ if (transition.phase === "restoring_previous" || transition.phase === "retiring_failed_candidate") {
1640
+ return await this.compensatePreCommitActivationFailure()
1641
+ }
1630
1642
  const release = this.releases.get(transition.candidateReleaseId)
1631
1643
  const previousRelease = transition.previousReleaseId ? this.releases.get(transition.previousReleaseId) : undefined
1632
1644
 
@@ -1668,7 +1680,23 @@ export default class RollbridgeDaemon {
1668
1680
 
1669
1681
  await this.failGenerationTransition(failure)
1670
1682
  this.logger("release generation activation failed", {error: failure, releaseId: release.releaseId})
1671
- throw error
1683
+ if (!previousRelease || !activationLifecycle) throw error
1684
+
1685
+ let compensation
1686
+
1687
+ try {
1688
+ compensation = await this.compensatePreCommitActivationFailure()
1689
+ } catch (compensationError) {
1690
+ throw new AggregateError(
1691
+ [error, compensationError],
1692
+ `${failure}; pre-commit compensation failed: ${compensationError instanceof Error ? compensationError.message : String(compensationError)}`,
1693
+ {cause: compensationError}
1694
+ )
1695
+ }
1696
+ const compensatedError = /** @type {Error & {compensation?: Record<string, JsonValue>}} */ (new Error(`${failure}; compensation restored incumbent ${compensation.previousReleaseId} as authoritative and retired failed candidate ${compensation.candidateReleaseId}`, {cause: error}))
1697
+
1698
+ compensatedError.compensation = compensation
1699
+ throw compensatedError
1672
1700
  }
1673
1701
 
1674
1702
  // Activation acknowledgement and the logical proxy commit deliberately share one
@@ -1732,6 +1760,155 @@ export default class RollbridgeDaemon {
1732
1760
  }
1733
1761
  }
1734
1762
 
1763
+ /**
1764
+ * Restores the authoritative proxy target and retires only the failed candidate.
1765
+ * Every external effect is preceded by a durable phase checkpoint so recovery can
1766
+ * safely replay the paired lifecycle hooks, which are already idempotent contracts.
1767
+ * @returns {Promise<Record<string, JsonValue>>} Structured recovery result.
1768
+ */
1769
+ async compensatePreCommitActivationFailure() {
1770
+ const transition = this.generationTransition
1771
+
1772
+ if (!transition) throw new Error("No release generation transition to compensate")
1773
+ if (transition.phase !== "activating_candidate" && transition.phase !== "restoring_previous" && transition.phase !== "retiring_failed_candidate") {
1774
+ throw new Error(`Generation transition ${transition.candidateReleaseId} at ${transition.phase} is not safe for pre-commit compensation`)
1775
+ }
1776
+ if (!transition.previousReleaseId) throw new Error("Pre-commit compensation requires a retained previous release")
1777
+ const candidate = this.releases.get(transition.candidateReleaseId)
1778
+ const previous = this.releases.get(transition.previousReleaseId)
1779
+
1780
+ if (!candidate) throw new Error(`Generation transition candidate ${transition.candidateReleaseId} is not retained`)
1781
+ if (!previous) throw new Error(`Generation transition previous release ${transition.previousReleaseId} is not retained`)
1782
+ if (this.activeRelease !== previous) throw new Error(`Previous release ${previous.releaseId} is not the authoritative proxy target`)
1783
+
1784
+ if (transition.phase === "activating_candidate") {
1785
+ if (!transition.error && !transition.activationError) throw new Error("Candidate activation has no recorded failure to compensate")
1786
+ transition.activationError = transition.activationError || transition.error
1787
+ transition.compensationError = undefined
1788
+ await this.updateGenerationTransition("retiring_failed_candidate")
1789
+ }
1790
+
1791
+ if (transition.phase === "retiring_failed_candidate") {
1792
+ try {
1793
+ await candidate.beginRetirement(candidate.config, {retry: true})
1794
+ const candidateServiceIds = [...this.serviceReleaseIds.entries()]
1795
+ .filter(([, releaseId]) => releaseId === candidate.releaseId)
1796
+ .map(([serviceId]) => serviceId)
1797
+
1798
+ await this.stopStartedServices(candidateServiceIds)
1799
+ } catch (error) {
1800
+ const failure = error instanceof Error ? error.message : String(error)
1801
+
1802
+ transition.compensationError = failure
1803
+ await this.failGenerationTransition(`Failed candidate ${candidate.releaseId} retirement failed: ${failure}`)
1804
+ this.logger("release generation compensation candidate retirement failed", {
1805
+ activationError: transition.activationError,
1806
+ error: failure,
1807
+ releaseId: candidate.releaseId
1808
+ })
1809
+ throw new Error(`failed candidate ${candidate.releaseId} retirement failed: ${failure}`, {cause: error})
1810
+ }
1811
+ transition.compensationError = undefined
1812
+ await this.updateGenerationTransition("restoring_previous")
1813
+ this.logger("release generation compensation candidate retired", {releaseId: candidate.releaseId})
1814
+ }
1815
+
1816
+ if (transition.phase === "restoring_previous") {
1817
+ if (candidate.state !== "draining") throw new Error(`Failed candidate ${candidate.releaseId} is not retired before incumbent restoration`)
1818
+ try {
1819
+ await previous.reactivateGeneration()
1820
+ } catch (error) {
1821
+ const failure = error instanceof Error ? error.message : String(error)
1822
+
1823
+ transition.compensationError = failure
1824
+ await this.failGenerationTransition(`Incumbent ${previous.releaseId} restoration failed: ${failure}`)
1825
+ this.logger("release generation compensation restoration failed", {
1826
+ activationError: transition.activationError,
1827
+ error: failure,
1828
+ releaseId: previous.releaseId
1829
+ })
1830
+ throw new Error(`incumbent ${previous.releaseId} restoration failed: ${failure}`, {cause: error})
1831
+ }
1832
+ this.activeRelease = previous
1833
+ transition.compensationError = undefined
1834
+ this.logger("release generation compensation incumbent restored", {releaseId: previous.releaseId})
1835
+ }
1836
+
1837
+ const result = /** @type {Record<string, JsonValue>} */ ({
1838
+ activeReleaseId: previous.releaseId,
1839
+ activationError: transition.activationError,
1840
+ candidateReleaseId: candidate.releaseId,
1841
+ failedCandidateStatus: candidate.state,
1842
+ previousReleaseId: previous.releaseId,
1843
+ recoveryStatus: "recovered"
1844
+ })
1845
+
1846
+ this.config = previous.config
1847
+ this.generationTransition = undefined
1848
+ try {
1849
+ await this.checkpointGenerationTransition()
1850
+ } catch (error) {
1851
+ const failure = error instanceof Error ? error.message : String(error)
1852
+
1853
+ this.generationTransition = transition
1854
+ transition.compensationError = failure
1855
+ transition.error = `Compensation checkpoint clear failed: ${failure}`
1856
+ transition.journalRevision = (transition.journalRevision ?? 0) + 1
1857
+ transition.updatedAt = new Date().toISOString()
1858
+ await this.publishOwnerState().catch((publishError) => {
1859
+ this.logger("release generation compensation fence republish failed", {
1860
+ error: publishError instanceof Error ? publishError.message : String(publishError),
1861
+ releaseId: candidate.releaseId
1862
+ })
1863
+ })
1864
+ this.logger("release generation compensation checkpoint failed", {error: failure, releaseId: candidate.releaseId})
1865
+ throw new Error(`compensation checkpoint clear failed: ${failure}`, {cause: error})
1866
+ }
1867
+ void this.drainAndPrune(candidate, candidate.config)
1868
+ this.logger("release generation compensation completed", result)
1869
+ return result
1870
+ }
1871
+
1872
+ /**
1873
+ * Authenticated control recovery for one exact failed pre-commit transition.
1874
+ * @param {{previousReleaseId: string, releaseId: string, releasePath: string, revision: string}} identity - Exact transition fence.
1875
+ * @returns {Promise<Record<string, JsonValue>>} Structured recovery result.
1876
+ */
1877
+ async recoverGenerationTransition(identity) {
1878
+ const transition = this.generationTransition
1879
+
1880
+ if (!transition) {
1881
+ if (this.activeRelease?.releaseId !== identity.previousReleaseId || this.activeRelease.releaseId === identity.releaseId) {
1882
+ throw new Error("No matching recovered generation transition; the expected incumbent is not authoritative")
1883
+ }
1884
+ return {
1885
+ activeReleaseId: this.activeRelease.releaseId,
1886
+ candidateReleaseId: identity.releaseId,
1887
+ previousReleaseId: identity.previousReleaseId,
1888
+ recoveryStatus: "already_recovered"
1889
+ }
1890
+ }
1891
+ if (transition.previousReleaseId !== identity.previousReleaseId) {
1892
+ throw new Error(`Generation transition previous release is ${transition.previousReleaseId}; refusing stale recovery for ${identity.previousReleaseId}`)
1893
+ }
1894
+ if (transition.phase !== "activating_candidate" && transition.phase !== "restoring_previous" && transition.phase !== "retiring_failed_candidate") {
1895
+ throw new Error(`Generation transition ${transition.candidateReleaseId} at ${transition.phase} is not a safe failed pre-commit transition`)
1896
+ }
1897
+ const nextConfig = this.configPath ? await loadConfig(this.configPath) : this.config
1898
+
1899
+ this.assertReloadCompatible(nextConfig)
1900
+ this.assertExactGenerationTransition(transition, {
1901
+ config: nextConfig,
1902
+ releaseId: identity.releaseId,
1903
+ releasePath: identity.releasePath,
1904
+ revision: identity.revision
1905
+ })
1906
+ if (this.activeRelease?.releaseId !== transition.previousReleaseId) {
1907
+ throw new Error(`Previous release ${transition.previousReleaseId} is not the authoritative proxy target`)
1908
+ }
1909
+ return await this.compensatePreCommitActivationFailure()
1910
+ }
1911
+
1735
1912
  /**
1736
1913
  * @param {GenerationTransition} transition - Pending or committed exact transition.
1737
1914
  * @param {{config: import("./config.js").RollbridgeConfig, releaseId: string, releasePath: string, revision: string}} candidate - Requested identity.
@@ -23,6 +23,7 @@ export default class GuardianClient {
23
23
  this.processes = /** @type {Map<string, GuardianProcess>} */ (new Map())
24
24
  this.reservedProcessKey = /** @type {string | undefined} */ (undefined)
25
25
  this.reservedProcessProvenance = /** @type {string | undefined} */ (undefined)
26
+ this.generationReactivation = /** @type {boolean | undefined} */ (undefined)
26
27
  this.events = /** @type {Map<string, {reject: (error: Error) => void, resolve: (value: Record<string, import("./json.js").JsonValue>) => void}[]>} */ (new Map())
27
28
  this.eventHandlers = /** @type {Map<string, ((event: Record<string, import("./json.js").JsonValue>) => void)[]>} */ (new Map())
28
29
  }
@@ -51,6 +52,7 @@ export default class GuardianClient {
51
52
  if (child.connected) await new Promise((resolve) => child.once("disconnect", () => resolve(undefined)))
52
53
  child.unref()
53
54
  await this.connect()
55
+ this.generationReactivation = true
54
56
  }
55
57
 
56
58
  /**
@@ -184,9 +186,12 @@ export default class GuardianClient {
184
186
  await this.request({command: "owner-ready", ownerPid: process.pid})
185
187
  }
186
188
 
187
- /** @returns {Promise<{daemonRecovery: number}>} Guardian protocol capabilities. */
189
+ /** @returns {Promise<{daemonRecovery: number, generationReactivation?: number}>} Guardian protocol capabilities. */
188
190
  async capabilities() {
189
- return /** @type {{daemonRecovery: number}} */ (await this.request({command: "capabilities"}))
191
+ const capabilities = /** @type {{daemonRecovery: number, generationReactivation?: number}} */ (await this.request({command: "capabilities"}))
192
+
193
+ this.generationReactivation = capabilities.generationReactivation === 1
194
+ return capabilities
190
195
  }
191
196
 
192
197
  /** Starts graceful process retirement and relinquishes committed owner authority. */
@@ -445,6 +450,7 @@ class GuardianProcess extends ManagedProcess {
445
450
  this.cachedStatus = super.status()
446
451
  this.registration = /** @type {Promise<void> | undefined} */ (undefined)
447
452
  this.pendingUpdate = Promise.resolve()
453
+ this.compatibilityReactivated = false
448
454
  }
449
455
 
450
456
  async ensureRegistered() {
@@ -479,6 +485,7 @@ class GuardianProcess extends ManagedProcess {
479
485
  await this.pendingUpdate
480
486
  if (lifecycleRole) this.lifecycleRole = lifecycleRole
481
487
  this.cachedStatus = asProcessStatus(await this.client.request({command: "start", key: this.key, lifecycleRole, reason}))
488
+ if (this.cachedStatus.state === "running") this.compatibilityReactivated = false
482
489
  }
483
490
 
484
491
  /**
@@ -529,12 +536,52 @@ class GuardianProcess extends ManagedProcess {
529
536
  this.lifecycleRole = "active"
530
537
  }
531
538
 
539
+ async reactivateStrict() {
540
+ await this.ensureRegistered()
541
+ await this.pendingUpdate
542
+ const command = this.lifecycle.reactivateCommand ? "reactivate-with-command" : "reactivate"
543
+ try {
544
+ this.cachedStatus = asProcessStatus(await this.client.request({command, key: this.key}))
545
+ this.client.generationReactivation = true
546
+ this.compatibilityReactivated = false
547
+ } catch (error) {
548
+ if (!(error instanceof Error) || error.message !== `Unknown guardian command: ${command}`) throw error
549
+ this.client.generationReactivation = false
550
+ await this.adoptRetainedActive({activate: true})
551
+ }
552
+ this.lifecycleRole = "active"
553
+ }
554
+
555
+ /**
556
+ * Bridges a retained quiesced process owned by a pre-reactivation guardian.
557
+ * @param {{activate: boolean}} options - Whether to run the external activation acknowledgement.
558
+ */
559
+ async adoptRetainedActive({activate}) {
560
+ const before = this.cachedStatus
561
+
562
+ if (!before.pid || (before.state !== "quiesced" && before.state !== "running")) throw new Error(`Process ${this.id} is not retained for compatible reactivation`)
563
+ if (activate) await this.runReactivationHook(before.pid)
564
+ const verified = asProcessStatus(await this.client.request({command: "status", key: this.key}))
565
+
566
+ if (verified.pid !== before.pid || (verified.state !== "quiesced" && verified.state !== "running")) {
567
+ throw new Error(`Process ${this.id} exited before compatible reactivation completed`)
568
+ }
569
+ const adopted = asProcessStatus(await this.client.request({command: "start", key: this.key, lifecycleRole: "active", reason: "deploy"}))
570
+
571
+ if (adopted.pid !== before.pid) throw new Error(`Process ${this.id} changed before compatible reactivation was adopted`)
572
+ this.cachedStatus = asProcessStatus({...adopted, lifecycleRole: "active", state: "running"})
573
+ this.compatibilityReactivated = true
574
+ }
575
+
532
576
  /** @param {import("./managed-process.js").LifecycleRole} role - Exact generation role. */
533
577
  async setLifecycleRole(role) {
534
578
  await this.ensureRegistered()
535
579
  await this.pendingUpdate
536
580
  this.cachedStatus = asProcessStatus(await this.client.request({command: "set-lifecycle-role", key: this.key, lifecycleRole: role}))
537
581
  this.lifecycleRole = role
582
+ if (role === "active" && this.client.generationReactivation === false && this.cachedStatus.state === "quiesced") {
583
+ await this.adoptRetainedActive({activate: false})
584
+ }
538
585
  }
539
586
 
540
587
  async stop(options = {}) {
@@ -550,6 +597,12 @@ class GuardianProcess extends ManagedProcess {
550
597
  /** @param {Record<string, import("./json.js").JsonValue>} event - Guardian event. */
551
598
  onGuardianEvent(event) {
552
599
  if (event.status) this.cachedStatus = asProcessStatus(event.status)
600
+ if (this.compatibilityReactivated && (this.cachedStatus.state === "failed" || this.cachedStatus.state === "stopped") && this.shouldRestart()) {
601
+ this.compatibilityReactivated = false
602
+ void this.start("crash", this.lifecycleRole).catch((error) => {
603
+ this.logger("compatible reactivated process restart failed", {error: error instanceof Error ? error.message : String(error), id: this.id})
604
+ })
605
+ }
553
606
  if (event.event === "process-log") {
554
607
  const entry = asProcessLog(event.entry)
555
608
 
@@ -4,7 +4,7 @@ import {EventEmitter} from "node:events"
4
4
  import {spawn} from "node:child_process"
5
5
  import {processGroupHasLiveMembers, processGroupMembers} from "./process-memory.js"
6
6
 
7
- const ACTIVATION_HOOK_TIMEOUT_MS = 30000
7
+ const DEFAULT_ACTIVATION_HOOK_TIMEOUT_MS = 30000
8
8
  const MAX_BUFFERED_OUTPUT_CHARACTERS = 64 * 1024
9
9
 
10
10
  /**
@@ -486,14 +486,14 @@ export default class ManagedProcess extends EventEmitter {
486
486
  if (this.lifecycleRestoreBarrier) await this.lifecycleRestoreBarrier
487
487
  if (!this.child?.pid) {
488
488
  this.state = "stopped"
489
- if (this.lifecycle.activateCommand) this.lifecycleRole = "retired"
489
+ if (this.hasRestorableLifecycle()) this.lifecycleRole = "retired"
490
490
  return
491
491
  }
492
492
  this.state = "stopping"
493
493
  if (this.lifecycle.quietCommand) this.quiesceError = await this.runHook(this.lifecycle.quietCommand, this.hookTimeoutMs(), "quiet command")
494
494
  if (!this.quiesceError) {
495
495
  this.state = "quiesced"
496
- if (this.lifecycle.activateCommand) this.lifecycleRole = "retired"
496
+ if (this.hasRestorableLifecycle()) this.lifecycleRole = "retired"
497
497
  }
498
498
  })()
499
499
  return await this.quiescePromise
@@ -521,13 +521,65 @@ export default class ManagedProcess extends EventEmitter {
521
521
  const pid = this.pid
522
522
 
523
523
  if (!child?.pid || child.pid !== pid || this.state !== "running") throw new Error(`Process ${this.id} is not running for activation`)
524
- const error = await this.runHook(command, ACTIVATION_HOOK_TIMEOUT_MS, "activate command", pid)
525
-
526
- if (error) throw error
524
+ await this.runActivationHook(pid)
527
525
  if (this.child !== child || this.pid !== pid || this.state !== "running") throw new Error(`Process ${this.id} exited before activation completed`)
528
526
  this.lifecycleRole = "active"
529
527
  }
530
528
 
529
+ /** Reactivates an explicitly retired process without replacing its release-scoped process. */
530
+ async reactivateStrict() {
531
+ const child = this.child
532
+ const pid = this.pid
533
+
534
+ if (!child?.pid || child.pid !== pid || (this.state !== "quiesced" && this.state !== "running")) {
535
+ throw new Error(`Process ${this.id} is not retained for reactivation`)
536
+ }
537
+ await this.runReactivationHook(pid)
538
+ if (this.child !== child || this.pid !== pid) throw new Error(`Process ${this.id} exited before reactivation completed`)
539
+ this.intentionalStop = false
540
+ this.intentionalStopSignal = undefined
541
+ this.quiescePromise = undefined
542
+ this.quiesceError = undefined
543
+ this.state = "running"
544
+ this.lifecycleRole = "active"
545
+ this.startMemoryMonitor()
546
+ }
547
+
548
+ /**
549
+ * Runs the generation activation hook against one exact retained process.
550
+ * @param {number | undefined} pid - Exact process group leader.
551
+ * @returns {Promise<void>} Resolves after acknowledgement.
552
+ */
553
+ async runActivationHook(pid) {
554
+ const command = this.lifecycle.activateCommand
555
+
556
+ if (!command) return
557
+ const error = await this.runHook(command, this.activationHookTimeoutMs(), "activate command", pid)
558
+
559
+ if (error) throw error
560
+ }
561
+
562
+ /**
563
+ * Runs the process-specific reactivation hook, falling back to the generation
564
+ * coordinator's activation hook for the original lifecycle contract.
565
+ * @param {number | undefined} pid - Exact process group leader.
566
+ * @returns {Promise<void>} Resolves after acknowledgement.
567
+ */
568
+ async runReactivationHook(pid) {
569
+ const command = this.lifecycle.reactivateCommand ?? this.lifecycle.activateCommand
570
+
571
+ if (!command) return
572
+ const label = this.lifecycle.reactivateCommand ? "reactivate command" : "activate command"
573
+ const error = await this.runHook(command, this.activationHookTimeoutMs(), label, pid)
574
+
575
+ if (error) throw error
576
+ }
577
+
578
+ /** @returns {boolean} Whether this process owns a durable external lifecycle role. */
579
+ hasRestorableLifecycle() {
580
+ return Boolean(this.lifecycle.activateCommand || this.lifecycle.reactivateCommand)
581
+ }
582
+
531
583
  /**
532
584
  * Records the durable desired role without firing a lifecycle command.
533
585
  * @param {LifecycleRole} role - Exact role owned by this process generation.
@@ -538,16 +590,24 @@ export default class ManagedProcess extends EventEmitter {
538
590
 
539
591
  /** Restores an active or retired role after this exact process starts. */
540
592
  async restoreLifecycleRole() {
541
- if (!this.lifecycle.activateCommand || this.lifecycleRole === "candidate") return
542
- const command = this.lifecycleRole === "active" ? this.lifecycle.activateCommand : this.lifecycle.quietCommand
593
+ if (!this.hasRestorableLifecycle() || this.lifecycleRole === "candidate") return
594
+ const command = this.lifecycleRole === "active"
595
+ ? this.lifecycle.reactivateCommand ?? this.lifecycle.activateCommand
596
+ : this.lifecycle.quietCommand
543
597
 
544
598
  if (!command) throw new Error(`Process ${this.id} cannot restore lifecycle role ${this.lifecycleRole} without its paired command`)
545
- const timeoutMs = this.lifecycleRole === "active" ? ACTIVATION_HOOK_TIMEOUT_MS : this.hookTimeoutMs()
546
- const error = await this.runHook(command, timeoutMs, `${this.lifecycleRole === "active" ? "activate" : "quiet"} command`, this.pid)
599
+ const timeoutMs = this.lifecycleRole === "active" ? this.activationHookTimeoutMs() : this.hookTimeoutMs()
600
+ const activeLabel = this.lifecycle.reactivateCommand ? "reactivate" : "activate"
601
+ const error = await this.runHook(command, timeoutMs, `${this.lifecycleRole === "active" ? activeLabel : "quiet"} command`, this.pid)
547
602
 
548
603
  if (error) throw error
549
604
  }
550
605
 
606
+ /** @returns {number} Timeout used for activation/reactivation hooks. */
607
+ activationHookTimeoutMs() {
608
+ return this.lifecycle.activateTimeoutMs ?? DEFAULT_ACTIVATION_HOOK_TIMEOUT_MS
609
+ }
610
+
551
611
  /** @returns {number} Timeout used for lifecycle hooks. */
552
612
  hookTimeoutMs() {
553
613
  if (this.stopTimeoutMs === "indefinite") return 30000
@@ -760,7 +820,7 @@ export default class ManagedProcess extends EventEmitter {
760
820
  id: this.id,
761
821
  lastMemoryRestartAt: this.lastMemoryRestartAtMs === undefined ? undefined : new Date(this.lastMemoryRestartAtMs).toISOString(),
762
822
  lastStartReason: this.lastStartReason,
763
- ...(this.lifecycle.activateCommand ? {lifecycleRole: this.lifecycleRole} : {}),
823
+ ...(this.hasRestorableLifecycle() ? {lifecycleRole: this.lifecycleRole} : {}),
764
824
  logs: this.logs.slice(-this.outputLines),
765
825
  memoryRestarts: this.memoryRestarts,
766
826
  pid: this.pid,