rollbridge 0.1.56 → 0.1.59

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/config.js CHANGED
@@ -24,7 +24,12 @@ import {pathToFileURL} from "node:url"
24
24
  * @typedef {{forceStopTimeoutMs: number, processes: LegacyTakeoverProcessConfig[], screens: string[]}} LegacyTakeoverConfig
25
25
  * @typedef {{keep: number, maxAgeMs: number}} ReleaseRetentionConfig
26
26
  * @typedef {{reconnectGraceMs: number}} OwnerRecoveryConfig
27
- * @typedef {{application: string, control: ControlConfig, legacyTakeover?: LegacyTakeoverConfig, ownerRecovery?: OwnerRecoveryConfig, processes: ProcessConfig[], proxy: ProxyConfig, releaseRetention: ReleaseRetentionConfig, statePath?: string}} RollbridgeConfig
27
+ * @typedef {{command: string | undefined, timeoutMs: number}} ComposeDrainConfig
28
+ * @typedef {{command: string | undefined, intervalMs: number, timeoutMs: number}} ComposeInFlightConfig
29
+ * @typedef {{healthcheck: boolean, intervalMs: number, readyCommand: string | undefined, readyUrl: string | undefined, startDelayMs: number, timeoutMs: number}} ComposeActivationConfig
30
+ * @typedef {{enabled: boolean}} ComposeRollbackConfig
31
+ * @typedef {{activation: ComposeActivationConfig, composeFile: string, drain: ComposeDrainConfig, envFile: string | undefined, id: string, image: string, inFlight: ComposeInFlightConfig, projectName: string | undefined, projectDirectory: string, recreateTimeoutMs: number, rollback: ComposeRollbackConfig, service: string, stopGracePeriodMs: number}} ComposeUnitConfig
32
+ * @typedef {{application: string, compose?: ComposeUnitConfig[], composeStatePath?: string, control: ControlConfig, legacyTakeover?: LegacyTakeoverConfig, ownerRecovery?: OwnerRecoveryConfig, processes: ProcessConfig[], proxy: ProxyConfig, releaseRetention: ReleaseRetentionConfig, statePath?: string}} RollbridgeConfig
28
33
  * @typedef {{fix: string, message: string}} ConfigIssue
29
34
  */
30
35
 
@@ -72,6 +77,7 @@ export async function loadConfig(configPath) {
72
77
 
73
78
  config.control.path = path.resolve(configDirectory, config.control.path)
74
79
  if (config.statePath) config.statePath = path.resolve(configDirectory, config.statePath)
80
+ if (config.composeStatePath) config.composeStatePath = path.resolve(configDirectory, config.composeStatePath)
75
81
  return config
76
82
  }
77
83
 
@@ -158,13 +164,32 @@ export function validateConfig(rawConfig, configPath = process.cwd()) {
158
164
  const ownerRecoverySource = source.ownerRecovery === undefined || source.ownerRecovery === null ? undefined : objectAt(source.ownerRecovery, "ownerRecovery", issues)
159
165
  const reconnectGraceMs = ownerRecoverySource ? normalizeNumber(ownerRecoverySource.reconnectGraceMs, "ownerRecovery.reconnectGraceMs", issues, {default: 30000}) : 30000
160
166
  const ownerRecovery = ownerRecoverySource ? {reconnectGraceMs: nonNegativeOrDefault(reconnectGraceMs, "ownerRecovery.reconnectGraceMs", issues, 30000, true)} : undefined
167
+ const compose = normalizeComposeUnits(source.compose, issues)
168
+ const composeStatePath = compose.length === 0 && source.composeStatePath === undefined && source.composeStatePath !== null
169
+ ? undefined
170
+ : normalizeString(source.composeStatePath, "composeStatePath", issues, {default: path.resolve(path.dirname(configPath), `rollbridge-${application}.compose.state.json`)})
161
171
 
162
172
  if (ownerRecovery && !statePath) issues.push({fix: "Configure statePath when ownerRecovery is enabled.", message: "ownerRecovery requires statePath"})
163
173
 
164
174
  validateProcessSet(processes, issues)
165
175
  validateActivationLifecycle(processes, ownerRecovery, statePath, issues)
176
+ validateComposeUnits(compose, processes, issues)
166
177
 
167
- return {config: {application, control, legacyTakeover, ownerRecovery, processes, proxy, releaseRetention, statePath}, issues}
178
+ return {
179
+ config: {
180
+ application,
181
+ ...(compose.length > 0 ? {compose} : {}),
182
+ ...(composeStatePath ? {composeStatePath} : {}),
183
+ control,
184
+ legacyTakeover,
185
+ ownerRecovery,
186
+ processes,
187
+ proxy,
188
+ releaseRetention,
189
+ statePath
190
+ },
191
+ issues
192
+ }
168
193
  }
169
194
 
170
195
  /**
@@ -518,6 +543,180 @@ function normalizeLegacyTakeoverProcess(value, index, issues) {
518
543
  }]
519
544
  }
520
545
 
546
+ /**
547
+ * @param {JsonValue} value - Raw Compose unit list.
548
+ * @param {ConfigIssue[]} issues - Issue collector.
549
+ * @returns {ComposeUnitConfig[]} Normalized Compose units (empty when the key is absent).
550
+ */
551
+ function normalizeComposeUnits(value, issues) {
552
+ if (value === undefined || value === null) return []
553
+
554
+ if (!Array.isArray(value)) {
555
+ issues.push({fix: "Set compose to a list of Compose service units, or remove the key.", message: "compose must be an array"})
556
+
557
+ return []
558
+ }
559
+
560
+ if (value.length === 0) {
561
+ issues.push({fix: "Define at least one compose unit, or remove the compose key.", message: "compose must define at least one unit"})
562
+
563
+ return []
564
+ }
565
+
566
+ return value.map((unitSource, index) => normalizeComposeUnit(unitSource, index, issues))
567
+ }
568
+
569
+ /**
570
+ * @param {JsonValue} value - Raw Compose unit.
571
+ * @param {number} index - Unit index.
572
+ * @param {ConfigIssue[]} issues - Issue collector.
573
+ * @returns {ComposeUnitConfig} Normalized Compose unit.
574
+ */
575
+ function normalizeComposeUnit(value, index, issues) {
576
+ const key = `compose[${index}]`
577
+
578
+ if (!isPlainObject(value)) {
579
+ issues.push({fix: `Define ${key} as a mapping with id, service, image, and projectDirectory.`, message: `${key} must be an object`})
580
+
581
+ return {
582
+ activation: {healthcheck: true, intervalMs: 1000, readyCommand: undefined, readyUrl: undefined, startDelayMs: 0, timeoutMs: 60000},
583
+ composeFile: "compose.yml",
584
+ drain: {command: undefined, timeoutMs: 0},
585
+ envFile: undefined,
586
+ id: "",
587
+ image: "",
588
+ inFlight: {command: undefined, intervalMs: 1000, timeoutMs: 0},
589
+ projectName: undefined,
590
+ projectDirectory: "",
591
+ recreateTimeoutMs: 120000,
592
+ rollback: {enabled: true},
593
+ service: "",
594
+ stopGracePeriodMs: 10000
595
+ }
596
+ }
597
+
598
+ const source = value
599
+ const stopGracePeriodMs = normalizePositiveNumber(source.stopGracePeriodMs, `${key}.stopGracePeriodMs`, issues, 10000)
600
+ const drainSource = source.drain === undefined || source.drain === null ? undefined : objectAt(source.drain, `${key}.drain`, issues, {})
601
+ const inFlightSource = source.inFlight === undefined || source.inFlight === null ? undefined : objectAt(source.inFlight, `${key}.inFlight`, issues, {})
602
+ const activationSource = source.activation === undefined || source.activation === null ? {} : objectAt(source.activation, `${key}.activation`, issues, {})
603
+ const rollbackSource = source.rollback === undefined || source.rollback === null ? undefined : objectAt(source.rollback, `${key}.rollback`, issues, {})
604
+
605
+ /** @type {ComposeUnitConfig} */
606
+ const unit = {
607
+ activation: {
608
+ healthcheck: normalizeBoolean(activationSource.healthcheck, `${key}.activation.healthcheck`, issues, true),
609
+ intervalMs: normalizePositiveNumber(activationSource.intervalMs, `${key}.activation.intervalMs`, issues, 1000),
610
+ readyCommand: activationSource.readyCommand === undefined ? undefined : normalizeString(activationSource.readyCommand, `${key}.activation.readyCommand`, issues, {nonEmpty: true}),
611
+ readyUrl: activationSource.readyUrl === undefined ? undefined : normalizeString(activationSource.readyUrl, `${key}.activation.readyUrl`, issues, {nonEmpty: true}),
612
+ startDelayMs: normalizeStartDelayMs(activationSource.startDelayMs, `${key}.activation.startDelayMs`, issues),
613
+ timeoutMs: normalizePositiveNumber(activationSource.timeoutMs, `${key}.activation.timeoutMs`, issues, 60000)
614
+ },
615
+ composeFile: normalizeString(source.composeFile, `${key}.composeFile`, issues, {default: "compose.yml"}),
616
+ drain: {
617
+ command: drainSource?.command === undefined ? undefined : normalizeString(drainSource.command, `${key}.drain.command`, issues, {nonEmpty: true}),
618
+ timeoutMs: normalizeNumber(drainSource?.timeoutMs, `${key}.drain.timeoutMs`, issues, {default: 60000})
619
+ },
620
+ envFile: source.envFile === undefined || source.envFile === null ? undefined : normalizeString(source.envFile, `${key}.envFile`, issues, {nonEmpty: true}),
621
+ id: normalizeString(source.id, `${key}.id`, issues),
622
+ image: normalizeString(source.image, `${key}.image`, issues),
623
+ inFlight: {
624
+ command: inFlightSource?.command === undefined ? undefined : normalizeString(inFlightSource.command, `${key}.inFlight.command`, issues, {nonEmpty: true}),
625
+ intervalMs: normalizePositiveNumber(inFlightSource?.intervalMs, `${key}.inFlight.intervalMs`, issues, 1000),
626
+ timeoutMs: normalizeNumber(inFlightSource?.timeoutMs, `${key}.inFlight.timeoutMs`, issues, {default: 0})
627
+ },
628
+ projectName: source.projectName === undefined || source.projectName === null ? undefined : normalizeString(source.projectName, `${key}.projectName`, issues, {nonEmpty: true}),
629
+ projectDirectory: normalizeString(source.projectDirectory, `${key}.projectDirectory`, issues),
630
+ recreateTimeoutMs: normalizePositiveNumber(source.recreateTimeoutMs, `${key}.recreateTimeoutMs`, issues, 120000),
631
+ rollback: {enabled: normalizeBoolean(rollbackSource?.enabled, `${key}.rollback.enabled`, issues, true)},
632
+ service: normalizeString(source.service, `${key}.service`, issues),
633
+ stopGracePeriodMs
634
+ }
635
+
636
+ if (unit.drain.command !== undefined && unit.drain.timeoutMs <= 0) {
637
+ issues.push({fix: `Set ${key}.drain.timeoutMs to a positive number of milliseconds; with 0 the quiescence step is skipped and ${key}.drain.command never runs.`, message: `${key}.drain.command requires a positive ${key}.drain.timeoutMs`})
638
+ }
639
+
640
+ if (unit.inFlight.command !== undefined && unit.inFlight.timeoutMs <= 0) {
641
+ issues.push({fix: `Set ${key}.inFlight.timeoutMs to how long the roll may wait for in-flight work, e.g. 120000; with 0 the gate is skipped and ${key}.inFlight.command never runs.`, message: `${key}.inFlight.command requires a positive ${key}.inFlight.timeoutMs`})
642
+ }
643
+
644
+ if (unit.image.length > 0 && !isComposeImageReference(unit.image)) {
645
+ issues.push({fix: `Set ${key}.image to an image reference pinned to a tag or digest, e.g. "registry.local/socketduct-gateway:1.4.0".`, message: `${key}.image "${unit.image}" is not a valid image reference`})
646
+ }
647
+
648
+ return unit
649
+ }
650
+
651
+ /**
652
+ * @param {JsonValue} value - Raw positive-number value.
653
+ * @param {string} key - Config key.
654
+ * @param {ConfigIssue[]} issues - Issue collector.
655
+ * @param {number} fallback - Default and fallback value.
656
+ * @returns {number} Positive number, or the fallback when invalid.
657
+ */
658
+ function normalizePositiveNumber(value, key, issues, fallback) {
659
+ const number = normalizeNumber(value, key, issues, {default: fallback})
660
+
661
+ if (!(number > 0)) {
662
+ issues.push({fix: `Set ${key} to a positive number of milliseconds, e.g. ${fallback}.`, message: `${key} must be a positive number`})
663
+
664
+ return fallback
665
+ }
666
+
667
+ return number
668
+ }
669
+
670
+ /**
671
+ * @param {string} image - Image reference.
672
+ * @returns {boolean} True when the reference is a usable Docker image name.
673
+ */
674
+ function isComposeImageReference(image) {
675
+ if (/\s/.test(image) || image.startsWith("-")) return false
676
+
677
+ const [name, digest] = image.split("@")
678
+
679
+ if (name.length === 0) return false
680
+ if (digest === undefined) return true
681
+
682
+ return /^sha256:[a-f0-9]{64}$/.test(digest)
683
+ }
684
+
685
+ /**
686
+ * Validates cross-unit rules: unique unit ids, no collision with process ids, and the fields a
687
+ * drain-first roll needs.
688
+ * @param {ComposeUnitConfig[]} compose - Normalized Compose units.
689
+ * @param {ProcessConfig[]} processes - Normalized processes.
690
+ * @param {ConfigIssue[]} issues - Issue collector.
691
+ * @returns {void}
692
+ */
693
+ function validateComposeUnits(compose, processes, issues) {
694
+ if (compose.length === 0) return
695
+
696
+ const processIds = new Set(processes.map((processConfig) => processConfig.id))
697
+ const seenIds = /** @type {Set<string>} */ (new Set())
698
+
699
+ for (const unit of compose) {
700
+ if (!unit.id) continue
701
+
702
+ if (seenIds.has(unit.id)) {
703
+ issues.push({fix: `Give each compose unit a unique id; "${unit.id}" is used more than once.`, message: `Duplicate compose unit id: ${unit.id}`})
704
+ }
705
+
706
+ seenIds.add(unit.id)
707
+
708
+ if (processIds.has(unit.id)) {
709
+ issues.push({fix: `Rename compose unit "${unit.id}" or the process with the same id; Rollbridge targets both by id.`, message: `Compose unit id "${unit.id}" collides with a process id`})
710
+ }
711
+ }
712
+
713
+ for (const [index, unit] of compose.entries()) {
714
+ if (!unit.service) issues.push({fix: `Set compose[${index}].service to the Compose service name to roll.`, message: `compose[${index}].service is required`})
715
+ if (!unit.projectDirectory) issues.push({fix: `Set compose[${index}].projectDirectory to the directory holding the Compose file.`, message: `compose[${index}].projectDirectory is required`})
716
+ if (!unit.image) issues.push({fix: `Set compose[${index}].image to the target image reference.`, message: `compose[${index}].image is required`})
717
+ }
718
+ }
719
+
521
720
  /**
522
721
  * @param {JsonValue} value - Raw stop signal.
523
722
  * @param {string} key - Config key.
package/src/daemon.js CHANGED
@@ -146,7 +146,7 @@ export default class RollbridgeDaemon {
146
146
  this.logger("release generation transition recovery failed", {error: error instanceof Error ? error.message : String(error), releaseId: transition.candidateReleaseId})
147
147
  }
148
148
  return {}
149
- })
149
+ }, {waitingFor: "the retained release generations to settle"})
150
150
  } finally {
151
151
  this.generationTransitionRecovery = false
152
152
  }
@@ -1389,7 +1389,7 @@ export default class RollbridgeDaemon {
1389
1389
  releaseId: stringOrUndefined(data.releaseId),
1390
1390
  releasePath: requiredString(data.releasePath, "releasePath"),
1391
1391
  revision: stringOrUndefined(data.revision)
1392
- }))
1392
+ }), {waitingFor: "candidate activation and the retired generation handoff"})
1393
1393
  }
1394
1394
 
1395
1395
  if (commandName === "recover-generation-transition") {
@@ -1399,7 +1399,7 @@ export default class RollbridgeDaemon {
1399
1399
  releaseId: requiredString(data.releaseId, "releaseId"),
1400
1400
  releasePath: requiredString(data.releasePath, "releasePath"),
1401
1401
  revision: requiredString(data.revision, "revision")
1402
- }))
1402
+ }), {waitingFor: "the retained release generations to settle"})
1403
1403
  }
1404
1404
 
1405
1405
  if (commandName === "status") {
@@ -1414,18 +1414,18 @@ export default class RollbridgeDaemon {
1414
1414
  return await this.executeOwnerMutation("stop", async () => {
1415
1415
  await this.stopRelease(stringOrUndefined(data.releaseId))
1416
1416
  return this.status()
1417
- })
1417
+ }, {waitingFor: "the release's managed processes to stop"})
1418
1418
  }
1419
1419
 
1420
1420
  if (commandName === "restart") {
1421
1421
  return await this.executeOwnerMutation("restart", async () => await this.restartProcesses({
1422
1422
  policy: stringOrUndefined(data.policy),
1423
1423
  processId: stringOrUndefined(data.processId)
1424
- }))
1424
+ }), {waitingFor: "the managed process to stop and start again"})
1425
1425
  }
1426
1426
 
1427
1427
  if (commandName === "rollback") {
1428
- return await this.executeOwnerMutation("rollback", async () => await this.rollback({releaseId: stringOrUndefined(data.releaseId)}))
1428
+ return await this.executeOwnerMutation("rollback", async () => await this.rollback({releaseId: stringOrUndefined(data.releaseId)}), {waitingFor: "the previous release to activate"})
1429
1429
  }
1430
1430
 
1431
1431
  if (commandName === "shutdown") {
@@ -1489,11 +1489,13 @@ export default class RollbridgeDaemon {
1489
1489
  * Runs one owner mutation under the guardian's prepare/commit exclusion fence.
1490
1490
  * @param {string} operation - Mutation diagnostic name.
1491
1491
  * @param {() => Promise<Record<string, JsonValue>>} callback - Mutating operation.
1492
+ * @param {{waitingFor?: string}} [options] - What this mutation blocks on, so a refused
1493
+ * later mutation names the real blocker instead of just "in progress".
1492
1494
  * @returns {Promise<Record<string, JsonValue>>} Operation result.
1493
1495
  */
1494
- async executeOwnerMutation(operation, callback) {
1496
+ async executeOwnerMutation(operation, callback, {waitingFor} = {}) {
1495
1497
  if (!this.guardian) return await callback()
1496
- const mutationId = await this.guardian.beginOwnerMutation(operation)
1498
+ const mutationId = await this.guardian.beginOwnerMutation(operation, {ownerPid: process.pid, waitingFor})
1497
1499
  let result
1498
1500
  let operationError
1499
1501
 
@@ -1946,6 +1948,19 @@ export default class RollbridgeDaemon {
1946
1948
  recoveryStatus: "already_recovered"
1947
1949
  }
1948
1950
  }
1951
+ // A committed journal describes the transition that already published the current
1952
+ // incumbent, not an in-flight transition to recover. When the recovery fence names
1953
+ // that live incumbent, the failed deploy never journaled its own transition, so the
1954
+ // only fence that describes the authoritative release is the completed one: recovery
1955
+ // is already done. In-flight phases keep the stale-fence guard failing closed.
1956
+ if (transition.phase === "committed" && transition.candidateReleaseId === identity.previousReleaseId && this.activeRelease?.releaseId === identity.previousReleaseId && this.releases.get(transition.candidateReleaseId) === this.activeRelease) {
1957
+ return {
1958
+ activeReleaseId: this.activeRelease.releaseId,
1959
+ candidateReleaseId: identity.releaseId,
1960
+ previousReleaseId: identity.previousReleaseId,
1961
+ recoveryStatus: "already_recovered"
1962
+ }
1963
+ }
1949
1964
  if (transition.previousReleaseId !== identity.previousReleaseId) {
1950
1965
  throw new Error(`Generation transition previous release is ${transition.previousReleaseId}; refusing stale recovery for ${identity.previousReleaseId}`)
1951
1966
  }
package/src/doctor.js CHANGED
@@ -5,6 +5,7 @@ import fs from "node:fs/promises"
5
5
  import net from "node:net"
6
6
  import path from "node:path"
7
7
  import {inspectControlSocket} from "./daemon.js"
8
+ import {composeGlobalArgs, runComposeCommand} from "./compose-unit.js"
8
9
  import {liveProcesses, readState} from "./state-store.js"
9
10
  import {processTemplateContext, renderObject, renderTemplate} from "./template.js"
10
11
 
@@ -36,9 +37,75 @@ export async function runEnvironmentChecks(config) {
36
37
  checks.push(await orphanCheck(config.statePath, daemonRunning))
37
38
  }
38
39
 
40
+ if (config.compose !== undefined) checks.push(...await runComposeChecks(config))
41
+
42
+ return checks
43
+ }
44
+
45
+ /**
46
+ * Pre-flights every declared Compose unit: the Docker Compose CLI, the unit's Compose file, and
47
+ * the service it names inside that file.
48
+ * @param {import("./config.js").RollbridgeConfig} config - Normalized config.
49
+ * @returns {Promise<DoctorCheck[]>} One check per probed aspect.
50
+ */
51
+ async function runComposeChecks(config) {
52
+ /** @type {DoctorCheck[]} */
53
+ const checks = []
54
+ const version = await runComposeCommand({args: ["docker", "compose", "version"], timeoutMs: 15000})
55
+
56
+ if (version.code !== 0) {
57
+ return [{detail: `docker compose is not usable: ${version.stderr.trim() || `exit ${version.code}`}`, name: "docker compose", ok: false}]
58
+ }
59
+
60
+ checks.push({detail: version.stdout.trim() || "available", name: "docker compose", ok: true})
61
+
62
+ for (const unit of config.compose ?? []) {
63
+ const composeFile = path.resolve(unit.projectDirectory, unit.composeFile)
64
+
65
+ if (!await fileExists(composeFile)) {
66
+ checks.push({detail: `${composeFile} is missing; unit "${unit.id}" cannot be rolled`, name: `compose unit ${unit.id}`, ok: false})
67
+ continue
68
+ }
69
+
70
+ if (unit.envFile !== undefined && !await fileExists(path.resolve(unit.projectDirectory, unit.envFile))) {
71
+ checks.push({detail: `${path.resolve(unit.projectDirectory, unit.envFile)} is missing; unit "${unit.id}" cannot be rolled`, name: `compose unit ${unit.id}`, ok: false})
72
+ continue
73
+ }
74
+
75
+ const services = await runComposeCommand({args: [...composeGlobalArgs(unit), "config", "--services"], cwd: unit.projectDirectory, timeoutMs: 30000})
76
+
77
+ if (services.code !== 0) {
78
+ checks.push({detail: `could not read ${composeFile}: ${services.stderr.trim() || `exit ${services.code}`}`, name: `compose unit ${unit.id}`, ok: false})
79
+ continue
80
+ }
81
+
82
+ const listed = services.stdout.split("\n").map((line) => line.trim()).filter(Boolean)
83
+
84
+ if (!listed.includes(unit.service)) {
85
+ checks.push({detail: `service "${unit.service}" is not declared in ${composeFile}; found ${listed.join(", ") || "no services"}`, name: `compose unit ${unit.id}`, ok: false})
86
+ continue
87
+ }
88
+
89
+ checks.push({detail: `${unit.service} in ${composeFile}, target image ${unit.image}`, name: `compose unit ${unit.id}`, ok: true})
90
+ }
91
+
39
92
  return checks
40
93
  }
41
94
 
95
+ /**
96
+ * @param {string} filePath - Path to probe.
97
+ * @returns {Promise<boolean>} True when the path exists.
98
+ */
99
+ async function fileExists(filePath) {
100
+ try {
101
+ await fs.access(filePath)
102
+
103
+ return true
104
+ } catch {
105
+ return false
106
+ }
107
+ }
108
+
42
109
  /**
43
110
  * @param {string} statePath - Configured state file path.
44
111
  * @returns {Promise<DoctorCheck>} Whether the state file's directory is writable.
@@ -323,10 +323,11 @@ export default class GuardianClient {
323
323
  /**
324
324
  * Acquires the committed owner's mutation fence.
325
325
  * @param {string} operation - Control mutation name.
326
+ * @param {{mutationTimeoutMs?: number, ownerPid?: number, waitingFor?: string}} [details] - Lease bound, holding PID and what the mutation blocks on.
326
327
  * @returns {Promise<string>} Mutation lease id.
327
328
  */
328
- async beginOwnerMutation(operation) {
329
- const result = /** @type {{mutationId: string}} */ (await this.request({command: "begin-owner-mutation", operation}))
329
+ async beginOwnerMutation(operation, {mutationTimeoutMs, ownerPid, waitingFor} = {}) {
330
+ const result = /** @type {{mutationId: string}} */ (await this.request({command: "begin-owner-mutation", mutationTimeoutMs, operation, ownerPid, waitingFor}))
330
331
 
331
332
  return result.mutationId
332
333
  }
@@ -336,6 +337,16 @@ export default class GuardianClient {
336
337
  await this.request({command: "end-owner-mutation", mutationId})
337
338
  }
338
339
 
340
+ /**
341
+ * Refines what a held mutation is waiting for, so a later refusal names the real blocker.
342
+ * @param {string} mutationId - Mutation lease id.
343
+ * @param {string} waitingFor - Current blocker.
344
+ * @returns {Promise<void>} Report acknowledged.
345
+ */
346
+ async reportOwnerMutationWait(mutationId, waitingFor) {
347
+ await this.request({command: "report-owner-mutation-wait", mutationId, waitingFor})
348
+ }
349
+
339
350
  /** @returns {Promise<{committedReplacementId: string | null, ownerClaimed: boolean, retirementFailed?: boolean, retirementPending?: boolean, retirementReady?: boolean}>} Transaction status. */
340
351
  async replacementStatus() {
341
352
  return /** @type {{committedReplacementId: string | null, ownerClaimed: boolean, retirementFailed?: boolean, retirementPending?: boolean, retirementReady?: boolean}} */ (await this.request({command: "replacement-status"}))
@@ -12,6 +12,10 @@ import ManagedProcess from "./managed-process.js"
12
12
  import {writeState} from "./state-store.js"
13
13
 
14
14
  const OWNER_RESTART_RETRY_MS = 1000
15
+ // An owner mutation is deploy/stop/restart/rollback/recovery work. A retained
16
+ // generation may legitimately drain for hours, but a mutation that holds the
17
+ // fence that long is stuck, so the lease is bounded and released on expiry.
18
+ const OWNER_MUTATION_LEASE_MS = 900000
15
19
 
16
20
  /** @typedef {import("node:child_process").ChildProcess["signalCode"]} ChildExitSignal */
17
21
 
@@ -32,7 +36,9 @@ const OWNER_RESTART_RETRY_MS = 1000
32
36
  * @property {import("./json.js").JsonValue} [ownerState] - Private owner transfer state.
33
37
  * @property {string} [replacementId] - Prepared replacement transaction id.
34
38
  * @property {string} [mutationId] - Owner mutation lease id.
39
+ * @property {number} [mutationTimeoutMs] - Bound for the owner mutation lease.
35
40
  * @property {string} [operation] - Owner mutation diagnostic name.
41
+ * @property {string} [waitingFor] - What the held owner mutation is blocked on.
36
42
  * @property {number} [ownerPid] - Exact claimant process PID.
37
43
  * @property {import("./json.js").JsonValue} [recoverySnapshot] - Public bridge recovery snapshot.
38
44
  * @property {string} [statePath] - Exact public recovery state path.
@@ -103,6 +109,19 @@ const legacyKeys = legacyGuardian ? committedOwnerProcessKeys(ownerState) : new
103
109
  let ownerMutationClient
104
110
  /** @type {string | undefined} */
105
111
  let ownerMutationId
112
+ /** @type {string | undefined} */
113
+ let ownerMutationOperation
114
+ /** @type {string | undefined} */
115
+ let ownerMutationWaitingFor
116
+ /** @type {number | undefined} */
117
+ let ownerMutationPid
118
+ /** @type {number | undefined} */
119
+ let ownerMutationStartedAt
120
+ let ownerMutationLeaseMs = OWNER_MUTATION_LEASE_MS
121
+ /** @type {ReturnType<typeof setTimeout> | undefined} */
122
+ let ownerMutationLeaseTimer
123
+ /** @type {{elapsedMs: number, leaseMs: number, mutationId: string, operation: string} | undefined} */
124
+ let ownerMutationExpired
106
125
  /** @type {net.Socket | undefined} */
107
126
  let retiringClient
108
127
  /** @type {string | undefined} */
@@ -154,10 +173,7 @@ const server = net.createServer((socket) => {
154
173
  clearTimeout(waiter.timer)
155
174
  waiter.reject(new Error("Owner claimant disconnected"))
156
175
  }
157
- if (ownerMutationClient === socket && ownerClient !== socket) {
158
- ownerMutationClient = undefined
159
- ownerMutationId = undefined
160
- }
176
+ if (ownerMutationClient === socket && ownerClient !== socket) clearOwnerMutationLease()
161
177
  if (controlLessRetirement) {
162
178
  const preCommitHandoff = replacementClient && ownerClient === socket && retiringReplacementId === replacementId
163
179
 
@@ -301,8 +317,7 @@ async function execute(request, socket) {
301
317
  rememberDisconnectedOwnerSuccessors(socket)
302
318
  ownerClient = undefined
303
319
  ownerClientPid = undefined
304
- ownerMutationClient = undefined
305
- ownerMutationId = undefined
320
+ clearOwnerMutationLease()
306
321
  ownerRevision += 1
307
322
  grantNextOwner()
308
323
  return {retired: true}
@@ -397,26 +412,48 @@ async function execute(request, socket) {
397
412
  if (request.command === "begin-owner-mutation") {
398
413
  requireOwner(socket, request.command)
399
414
  if (replacementClient) throw new Error(`Owner mutation ${request.operation || "operation"} is fenced while an owner replacement is prepared`)
400
- if (ownerMutationClient) throw new Error("Another owner mutation is already in progress")
415
+ const leaseMs = request.mutationTimeoutMs === undefined ? OWNER_MUTATION_LEASE_MS : request.mutationTimeoutMs
416
+
417
+ if (!Number.isInteger(leaseMs) || leaseMs <= 0) throw new Error("Owner mutation lease bound must be a positive number of milliseconds")
418
+ if (ownerMutationClient) throw new Error(`Another owner mutation is already in progress: ${describeOwnerMutationLease()}; refused ${request.operation || "operation"}`)
401
419
  ownerMutationClient = socket
402
420
  ownerMutationId = crypto.randomUUID()
403
- return {mutationId: ownerMutationId}
421
+ ownerMutationOperation = request.operation
422
+ ownerMutationWaitingFor = request.waitingFor
423
+ ownerMutationPid = request.ownerPid ?? ownerClientPid
424
+ ownerMutationStartedAt = Date.now()
425
+ ownerMutationLeaseMs = leaseMs
426
+ armOwnerMutationLease(ownerMutationId)
427
+ return {leaseMs, mutationId: ownerMutationId}
404
428
  }
405
429
 
406
430
  if (request.command === "end-owner-mutation") {
407
431
  requireOwner(socket, request.command)
432
+ if (ownerMutationExpired && request.mutationId === ownerMutationExpired.mutationId) {
433
+ const expired = ownerMutationExpired
434
+
435
+ ownerMutationExpired = undefined
436
+ throw new Error(`Owner mutation ${expired.operation} exceeded its ${expired.leaseMs} ms lease and was released after ${expired.elapsedMs} ms; the mutation failed and its lease is gone`)
437
+ }
408
438
  if (ownerMutationClient !== socket || request.mutationId !== ownerMutationId) throw new Error("Owner mutation lease does not match the active mutation")
409
- ownerMutationClient = undefined
410
- ownerMutationId = undefined
439
+ clearOwnerMutationLease()
411
440
  return {ended: true}
412
441
  }
413
442
 
443
+ if (request.command === "report-owner-mutation-wait") {
444
+ requireOwner(socket, request.command)
445
+ if (ownerMutationClient !== socket || request.mutationId !== ownerMutationId) throw new Error("Owner mutation lease does not match the active mutation")
446
+ if (typeof request.waitingFor !== "string" || !request.waitingFor.trim()) throw new Error("Owner mutation wait report requires a waitingFor description")
447
+ ownerMutationWaitingFor = request.waitingFor.trim()
448
+ return {reported: true}
449
+ }
450
+
414
451
  if (request.command === "prepare-owner-replacement") {
415
452
  if (retirementFailed) throw new Error("Guardian owner replacement is fenced after incomplete retired-listener state transfer")
416
453
  if (retiringClient) throw new Error("Guardian owner replacement cannot prepare while listener retirement is pending")
417
454
  if (socket === ownerClient) throw new Error("Committed owner cannot prepare itself as a replacement")
418
455
  if (replacementClient && replacementClient !== socket) throw new Error("Another owner replacement candidate is already prepared")
419
- if (ownerMutationClient) throw new Error("Owner replacement cannot prepare while an owner mutation is in progress")
456
+ if (ownerMutationClient) throw new Error(`Owner replacement cannot prepare while an owner mutation is in progress: ${describeOwnerMutationLease()}`)
420
457
  if (!ownerState) throw new Error("Committed owner has not published transferable state")
421
458
  const currentAuthority = ownerAuthority(ownerState)
422
459
  const resumesCommittedAuthority = !ownerClient && isDeepStrictEqual(currentAuthority, request.nextAuthority)
@@ -613,8 +650,18 @@ async function execute(request, socket) {
613
650
  const existing = processes.get(request.key)
614
651
 
615
652
  if (existing) {
616
- if (existing.provenance !== request.provenance) throw new Error(`Guardian provenance mismatch for ${request.key}`)
617
- return existing.process.status()
653
+ if (existing.provenance === request.provenance) return existing.process.status()
654
+ // A provenance mismatch only fences a process that still exists. When the registered
655
+ // child is conclusively gone (stopped or failed with no live pid), the old record is
656
+ // residue of a dead generation, and demanding its provenance wedges every later deploy
657
+ // against a process that no longer exists (TensorBuzz production, STC-695). The owner
658
+ // may re-register the key under the new definition; live processes stay fail-closed.
659
+ const existingStatus = existing.process.status()
660
+ const childIsGone = (existingStatus.state === "stopped" || existingStatus.state === "failed") && existingStatus.pid === undefined
661
+
662
+ if (!childIsGone) throw new Error(`Guardian provenance mismatch for ${request.key}`)
663
+ requireOwner(socket, request.command)
664
+ processes.delete(request.key)
618
665
  }
619
666
 
620
667
  const recoversLegacyProcess = Boolean(legacyGuardian && legacyKeys.has(request.key))
@@ -705,6 +752,82 @@ async function execute(request, socket) {
705
752
  return status
706
753
  }
707
754
 
755
+ /**
756
+ * Names the mutation holding the owner-mutation fence: which mutation, which process,
757
+ * since when, and what it is blocked on. Refusals must be diagnosable without a live
758
+ * inspection of the daemon.
759
+ * @returns {string} Holder description used in refusal messages.
760
+ */
761
+ function describeOwnerMutationLease() {
762
+ const startedAt = ownerMutationStartedAt ?? 0
763
+ const elapsedSeconds = Math.max(0, Math.round((Date.now() - startedAt) / 1000))
764
+ const holderPid = ownerMutationPid ?? ownerClientPid
765
+
766
+ return [
767
+ `owner mutation "${ownerMutationOperation || "unnamed operation"}"`,
768
+ `mutation ${ownerMutationId || "unknown"}`,
769
+ `held by process ${holderPid ?? "unknown"}`,
770
+ `since ${new Date(startedAt).toISOString()}`,
771
+ `${elapsedSeconds}s elapsed`,
772
+ `waiting for ${ownerMutationWaitingFor || "the operation to report completion"}`
773
+ ].join("; ")
774
+ }
775
+
776
+ /** @returns {void} */
777
+ function clearOwnerMutationLease() {
778
+ if (ownerMutationLeaseTimer) clearTimeout(ownerMutationLeaseTimer)
779
+ ownerMutationLeaseTimer = undefined
780
+ ownerMutationClient = undefined
781
+ ownerMutationId = undefined
782
+ ownerMutationOperation = undefined
783
+ ownerMutationWaitingFor = undefined
784
+ ownerMutationPid = undefined
785
+ ownerMutationStartedAt = undefined
786
+ ownerMutationLeaseMs = OWNER_MUTATION_LEASE_MS
787
+ }
788
+
789
+ /**
790
+ * Arms the lease bound for one mutation.
791
+ * @param {string} mutationId - Lease id the timer belongs to.
792
+ * @returns {void}
793
+ */
794
+ function armOwnerMutationLease(mutationId) {
795
+ if (ownerMutationLeaseTimer) clearTimeout(ownerMutationLeaseTimer)
796
+ ownerMutationLeaseTimer = setTimeout(() => expireOwnerMutationLease(mutationId), ownerMutationLeaseMs)
797
+ ownerMutationLeaseTimer.unref()
798
+ }
799
+
800
+ /**
801
+ * A mutation that outruns its lease is reported and released instead of holding the fence
802
+ * forever, so a stuck mutation cannot lock out every later deploy. The holder learns the
803
+ * lease is gone when it tries to end it.
804
+ * @param {string} mutationId - Lease id the armed timer belongs to.
805
+ * @returns {void}
806
+ */
807
+ function expireOwnerMutationLease(mutationId) {
808
+ ownerMutationLeaseTimer = undefined
809
+ if (!ownerMutationClient || ownerMutationId !== mutationId) return
810
+ const operation = ownerMutationOperation || "unnamed operation"
811
+ const leaseMs = ownerMutationLeaseMs
812
+ const elapsedMs = Math.max(0, Date.now() - (ownerMutationStartedAt ?? 0))
813
+ const diagnostic = {
814
+ code: "OWNER_MUTATION_LEASE_EXPIRED",
815
+ elapsedMs,
816
+ event: "owner-mutation-expired",
817
+ leaseMs,
818
+ mutationId,
819
+ operation,
820
+ pid: ownerMutationPid ?? ownerClientPid,
821
+ waitingFor: ownerMutationWaitingFor || "the operation to report completion"
822
+ }
823
+
824
+ process.stderr.write(`${JSON.stringify({at: new Date().toISOString(), ...diagnostic})}\n`)
825
+ if (!ownerMutationClient.destroyed) ownerMutationClient.write(`${JSON.stringify(diagnostic)}\n`)
826
+ broadcast(diagnostic)
827
+ ownerMutationExpired = {elapsedMs, leaseMs, mutationId, operation}
828
+ clearOwnerMutationLease()
829
+ }
830
+
708
831
  /**
709
832
  * @param {net.Socket} socket - Requesting client.
710
833
  * @param {string} command - Command name.
@@ -725,10 +848,7 @@ function releaseClosedOwner(socket) {
725
848
  rememberDisconnectedOwnerSuccessors(socket)
726
849
  ownerClient = undefined
727
850
  ownerClientPid = undefined
728
- if (ownerMutationClient === socket) {
729
- ownerMutationClient = undefined
730
- ownerMutationId = undefined
731
- }
851
+ if (ownerMutationClient === socket) clearOwnerMutationLease()
732
852
  if (replacementClient && replacementOwnerState) commitReplacement()
733
853
  else {
734
854
  if (incompleteRestartChild) {