rollbridge 0.1.58 → 0.1.59

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,20 @@
1
+ ### Fixed
2
+
3
+ - A refused owner mutation now names its holder. The refusal used to read
4
+ `Another owner mutation is already in progress` with no context, so an
5
+ operator could not tell which command held the fence, which process held it,
6
+ for how long, or what it was blocked on. The guardian now records the
7
+ mutation's operation, lease id, holding PID, start time and current blocker
8
+ (`waitingFor`, refinable while the mutation runs through
9
+ `reportOwnerMutationWait`) and reports them in every refusal, including the
10
+ replacement-prepare fence.
11
+
12
+ ### Changed
13
+
14
+ - The owner-mutation fence is bounded. A mutation that outruns its lease
15
+ (default 15 minutes, per-command override through `mutationTimeoutMs`) is
16
+ logged as `OWNER_MUTATION_LEASE_EXPIRED`, reported to the holder, and
17
+ released, so a stuck mutation can no longer lock out every later deploy. The
18
+ holder fails loudly when it next ends that mutation instead of silently
19
+ keeping the fence. Retained generation drains are unaffected: they are not
20
+ owner mutations and may still run for hours.
@@ -148,3 +148,27 @@ separate destructive `--takeover-owner` path. If
148
148
  `retirementError` is set,
149
149
  inspect the quiet-hook events. Rollbridge deliberately leaves that generation
150
150
  alive rather than signaling arbitrary PIDs or continuing its stop sequence.
151
+
152
+ ## A command is refused with "Another owner mutation is already in progress"
153
+
154
+ **Symptom.** `deploy`, `stop`, `restart`, `rollback` or
155
+ `recover-generation-transition` fails with
156
+ `Another owner mutation is already in progress: owner mutation "<operation>";
157
+ mutation <id>; held by process <pid>; since <ISO-8601>; <n>s elapsed; waiting
158
+ for <blocker>; refused <operation>`.
159
+
160
+ **Cause.** The committed daemon holds the guardian's owner-mutation fence for
161
+ one control mutation at a time, and a second mutation arrived while it was still
162
+ held.
163
+
164
+ **Fix.** Read the named holder before touching anything: the message states
165
+ which mutation holds the fence, its lease id, the PID of the process that
166
+ started it, when it started, how long it has run, and the blocker that holder
167
+ last reported (`waitingFor`, which a running mutation can refine through
168
+ `reportOwnerMutationWait`). That is the command to inspect — not the one that
169
+ was refused. A mutation that outruns its lease (15 minutes by default,
170
+ overridable per command with `mutationTimeoutMs`) is logged as
171
+ `OWNER_MUTATION_LEASE_EXPIRED`, reported to the holder, and released, so the
172
+ next mutation proceeds and the stuck one fails loudly when it next ends its
173
+ lease. A retained jobs generation is not an owner mutation and may still drain
174
+ for hours; only the fence is bounded.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "rollbridge",
3
- "version": "0.1.58",
3
+ "version": "0.1.59",
4
4
  "description": "Zero-downtime process supervisor and local traffic switcher for deploy-managed apps.",
5
5
  "keywords": [
6
6
  "deploy",
package/src/daemon.js CHANGED
@@ -146,7 +146,7 @@ export default class RollbridgeDaemon {
146
146
  this.logger("release generation transition recovery failed", {error: error instanceof Error ? error.message : String(error), releaseId: transition.candidateReleaseId})
147
147
  }
148
148
  return {}
149
- })
149
+ }, {waitingFor: "the retained release generations to settle"})
150
150
  } finally {
151
151
  this.generationTransitionRecovery = false
152
152
  }
@@ -1389,7 +1389,7 @@ export default class RollbridgeDaemon {
1389
1389
  releaseId: stringOrUndefined(data.releaseId),
1390
1390
  releasePath: requiredString(data.releasePath, "releasePath"),
1391
1391
  revision: stringOrUndefined(data.revision)
1392
- }))
1392
+ }), {waitingFor: "candidate activation and the retired generation handoff"})
1393
1393
  }
1394
1394
 
1395
1395
  if (commandName === "recover-generation-transition") {
@@ -1399,7 +1399,7 @@ export default class RollbridgeDaemon {
1399
1399
  releaseId: requiredString(data.releaseId, "releaseId"),
1400
1400
  releasePath: requiredString(data.releasePath, "releasePath"),
1401
1401
  revision: requiredString(data.revision, "revision")
1402
- }))
1402
+ }), {waitingFor: "the retained release generations to settle"})
1403
1403
  }
1404
1404
 
1405
1405
  if (commandName === "status") {
@@ -1414,18 +1414,18 @@ export default class RollbridgeDaemon {
1414
1414
  return await this.executeOwnerMutation("stop", async () => {
1415
1415
  await this.stopRelease(stringOrUndefined(data.releaseId))
1416
1416
  return this.status()
1417
- })
1417
+ }, {waitingFor: "the release's managed processes to stop"})
1418
1418
  }
1419
1419
 
1420
1420
  if (commandName === "restart") {
1421
1421
  return await this.executeOwnerMutation("restart", async () => await this.restartProcesses({
1422
1422
  policy: stringOrUndefined(data.policy),
1423
1423
  processId: stringOrUndefined(data.processId)
1424
- }))
1424
+ }), {waitingFor: "the managed process to stop and start again"})
1425
1425
  }
1426
1426
 
1427
1427
  if (commandName === "rollback") {
1428
- return await this.executeOwnerMutation("rollback", async () => await this.rollback({releaseId: stringOrUndefined(data.releaseId)}))
1428
+ return await this.executeOwnerMutation("rollback", async () => await this.rollback({releaseId: stringOrUndefined(data.releaseId)}), {waitingFor: "the previous release to activate"})
1429
1429
  }
1430
1430
 
1431
1431
  if (commandName === "shutdown") {
@@ -1489,11 +1489,13 @@ export default class RollbridgeDaemon {
1489
1489
  * Runs one owner mutation under the guardian's prepare/commit exclusion fence.
1490
1490
  * @param {string} operation - Mutation diagnostic name.
1491
1491
  * @param {() => Promise<Record<string, JsonValue>>} callback - Mutating operation.
1492
+ * @param {{waitingFor?: string}} [options] - What this mutation blocks on, so a refused
1493
+ * later mutation names the real blocker instead of just "in progress".
1492
1494
  * @returns {Promise<Record<string, JsonValue>>} Operation result.
1493
1495
  */
1494
- async executeOwnerMutation(operation, callback) {
1496
+ async executeOwnerMutation(operation, callback, {waitingFor} = {}) {
1495
1497
  if (!this.guardian) return await callback()
1496
- const mutationId = await this.guardian.beginOwnerMutation(operation)
1498
+ const mutationId = await this.guardian.beginOwnerMutation(operation, {ownerPid: process.pid, waitingFor})
1497
1499
  let result
1498
1500
  let operationError
1499
1501
 
@@ -323,10 +323,11 @@ export default class GuardianClient {
323
323
  /**
324
324
  * Acquires the committed owner's mutation fence.
325
325
  * @param {string} operation - Control mutation name.
326
+ * @param {{mutationTimeoutMs?: number, ownerPid?: number, waitingFor?: string}} [details] - Lease bound, holding PID and what the mutation blocks on.
326
327
  * @returns {Promise<string>} Mutation lease id.
327
328
  */
328
- async beginOwnerMutation(operation) {
329
- const result = /** @type {{mutationId: string}} */ (await this.request({command: "begin-owner-mutation", operation}))
329
+ async beginOwnerMutation(operation, {mutationTimeoutMs, ownerPid, waitingFor} = {}) {
330
+ const result = /** @type {{mutationId: string}} */ (await this.request({command: "begin-owner-mutation", mutationTimeoutMs, operation, ownerPid, waitingFor}))
330
331
 
331
332
  return result.mutationId
332
333
  }
@@ -336,6 +337,16 @@ export default class GuardianClient {
336
337
  await this.request({command: "end-owner-mutation", mutationId})
337
338
  }
338
339
 
340
+ /**
341
+ * Refines what a held mutation is waiting for, so a later refusal names the real blocker.
342
+ * @param {string} mutationId - Mutation lease id.
343
+ * @param {string} waitingFor - Current blocker.
344
+ * @returns {Promise<void>} Report acknowledged.
345
+ */
346
+ async reportOwnerMutationWait(mutationId, waitingFor) {
347
+ await this.request({command: "report-owner-mutation-wait", mutationId, waitingFor})
348
+ }
349
+
339
350
  /** @returns {Promise<{committedReplacementId: string | null, ownerClaimed: boolean, retirementFailed?: boolean, retirementPending?: boolean, retirementReady?: boolean}>} Transaction status. */
340
351
  async replacementStatus() {
341
352
  return /** @type {{committedReplacementId: string | null, ownerClaimed: boolean, retirementFailed?: boolean, retirementPending?: boolean, retirementReady?: boolean}} */ (await this.request({command: "replacement-status"}))
@@ -12,6 +12,10 @@ import ManagedProcess from "./managed-process.js"
12
12
  import {writeState} from "./state-store.js"
13
13
 
14
14
  const OWNER_RESTART_RETRY_MS = 1000
15
+ // An owner mutation is deploy/stop/restart/rollback/recovery work. A retained
16
+ // generation may legitimately drain for hours, but a mutation that holds the
17
+ // fence that long is stuck, so the lease is bounded and released on expiry.
18
+ const OWNER_MUTATION_LEASE_MS = 900000
15
19
 
16
20
  /** @typedef {import("node:child_process").ChildProcess["signalCode"]} ChildExitSignal */
17
21
 
@@ -32,7 +36,9 @@ const OWNER_RESTART_RETRY_MS = 1000
32
36
  * @property {import("./json.js").JsonValue} [ownerState] - Private owner transfer state.
33
37
  * @property {string} [replacementId] - Prepared replacement transaction id.
34
38
  * @property {string} [mutationId] - Owner mutation lease id.
39
+ * @property {number} [mutationTimeoutMs] - Bound for the owner mutation lease.
35
40
  * @property {string} [operation] - Owner mutation diagnostic name.
41
+ * @property {string} [waitingFor] - What the held owner mutation is blocked on.
36
42
  * @property {number} [ownerPid] - Exact claimant process PID.
37
43
  * @property {import("./json.js").JsonValue} [recoverySnapshot] - Public bridge recovery snapshot.
38
44
  * @property {string} [statePath] - Exact public recovery state path.
@@ -103,6 +109,19 @@ const legacyKeys = legacyGuardian ? committedOwnerProcessKeys(ownerState) : new
103
109
  let ownerMutationClient
104
110
  /** @type {string | undefined} */
105
111
  let ownerMutationId
112
+ /** @type {string | undefined} */
113
+ let ownerMutationOperation
114
+ /** @type {string | undefined} */
115
+ let ownerMutationWaitingFor
116
+ /** @type {number | undefined} */
117
+ let ownerMutationPid
118
+ /** @type {number | undefined} */
119
+ let ownerMutationStartedAt
120
+ let ownerMutationLeaseMs = OWNER_MUTATION_LEASE_MS
121
+ /** @type {ReturnType<typeof setTimeout> | undefined} */
122
+ let ownerMutationLeaseTimer
123
+ /** @type {{elapsedMs: number, leaseMs: number, mutationId: string, operation: string} | undefined} */
124
+ let ownerMutationExpired
106
125
  /** @type {net.Socket | undefined} */
107
126
  let retiringClient
108
127
  /** @type {string | undefined} */
@@ -154,10 +173,7 @@ const server = net.createServer((socket) => {
154
173
  clearTimeout(waiter.timer)
155
174
  waiter.reject(new Error("Owner claimant disconnected"))
156
175
  }
157
- if (ownerMutationClient === socket && ownerClient !== socket) {
158
- ownerMutationClient = undefined
159
- ownerMutationId = undefined
160
- }
176
+ if (ownerMutationClient === socket && ownerClient !== socket) clearOwnerMutationLease()
161
177
  if (controlLessRetirement) {
162
178
  const preCommitHandoff = replacementClient && ownerClient === socket && retiringReplacementId === replacementId
163
179
 
@@ -301,8 +317,7 @@ async function execute(request, socket) {
301
317
  rememberDisconnectedOwnerSuccessors(socket)
302
318
  ownerClient = undefined
303
319
  ownerClientPid = undefined
304
- ownerMutationClient = undefined
305
- ownerMutationId = undefined
320
+ clearOwnerMutationLease()
306
321
  ownerRevision += 1
307
322
  grantNextOwner()
308
323
  return {retired: true}
@@ -397,26 +412,48 @@ async function execute(request, socket) {
397
412
  if (request.command === "begin-owner-mutation") {
398
413
  requireOwner(socket, request.command)
399
414
  if (replacementClient) throw new Error(`Owner mutation ${request.operation || "operation"} is fenced while an owner replacement is prepared`)
400
- if (ownerMutationClient) throw new Error("Another owner mutation is already in progress")
415
+ const leaseMs = request.mutationTimeoutMs === undefined ? OWNER_MUTATION_LEASE_MS : request.mutationTimeoutMs
416
+
417
+ if (!Number.isInteger(leaseMs) || leaseMs <= 0) throw new Error("Owner mutation lease bound must be a positive number of milliseconds")
418
+ if (ownerMutationClient) throw new Error(`Another owner mutation is already in progress: ${describeOwnerMutationLease()}; refused ${request.operation || "operation"}`)
401
419
  ownerMutationClient = socket
402
420
  ownerMutationId = crypto.randomUUID()
403
- return {mutationId: ownerMutationId}
421
+ ownerMutationOperation = request.operation
422
+ ownerMutationWaitingFor = request.waitingFor
423
+ ownerMutationPid = request.ownerPid ?? ownerClientPid
424
+ ownerMutationStartedAt = Date.now()
425
+ ownerMutationLeaseMs = leaseMs
426
+ armOwnerMutationLease(ownerMutationId)
427
+ return {leaseMs, mutationId: ownerMutationId}
404
428
  }
405
429
 
406
430
  if (request.command === "end-owner-mutation") {
407
431
  requireOwner(socket, request.command)
432
+ if (ownerMutationExpired && request.mutationId === ownerMutationExpired.mutationId) {
433
+ const expired = ownerMutationExpired
434
+
435
+ ownerMutationExpired = undefined
436
+ throw new Error(`Owner mutation ${expired.operation} exceeded its ${expired.leaseMs} ms lease and was released after ${expired.elapsedMs} ms; the mutation failed and its lease is gone`)
437
+ }
408
438
  if (ownerMutationClient !== socket || request.mutationId !== ownerMutationId) throw new Error("Owner mutation lease does not match the active mutation")
409
- ownerMutationClient = undefined
410
- ownerMutationId = undefined
439
+ clearOwnerMutationLease()
411
440
  return {ended: true}
412
441
  }
413
442
 
443
+ if (request.command === "report-owner-mutation-wait") {
444
+ requireOwner(socket, request.command)
445
+ if (ownerMutationClient !== socket || request.mutationId !== ownerMutationId) throw new Error("Owner mutation lease does not match the active mutation")
446
+ if (typeof request.waitingFor !== "string" || !request.waitingFor.trim()) throw new Error("Owner mutation wait report requires a waitingFor description")
447
+ ownerMutationWaitingFor = request.waitingFor.trim()
448
+ return {reported: true}
449
+ }
450
+
414
451
  if (request.command === "prepare-owner-replacement") {
415
452
  if (retirementFailed) throw new Error("Guardian owner replacement is fenced after incomplete retired-listener state transfer")
416
453
  if (retiringClient) throw new Error("Guardian owner replacement cannot prepare while listener retirement is pending")
417
454
  if (socket === ownerClient) throw new Error("Committed owner cannot prepare itself as a replacement")
418
455
  if (replacementClient && replacementClient !== socket) throw new Error("Another owner replacement candidate is already prepared")
419
- if (ownerMutationClient) throw new Error("Owner replacement cannot prepare while an owner mutation is in progress")
456
+ if (ownerMutationClient) throw new Error(`Owner replacement cannot prepare while an owner mutation is in progress: ${describeOwnerMutationLease()}`)
420
457
  if (!ownerState) throw new Error("Committed owner has not published transferable state")
421
458
  const currentAuthority = ownerAuthority(ownerState)
422
459
  const resumesCommittedAuthority = !ownerClient && isDeepStrictEqual(currentAuthority, request.nextAuthority)
@@ -715,6 +752,82 @@ async function execute(request, socket) {
715
752
  return status
716
753
  }
717
754
 
755
+ /**
756
+ * Names the mutation holding the owner-mutation fence: which mutation, which process,
757
+ * since when, and what it is blocked on. Refusals must be diagnosable without a live
758
+ * inspection of the daemon.
759
+ * @returns {string} Holder description used in refusal messages.
760
+ */
761
+ function describeOwnerMutationLease() {
762
+ const startedAt = ownerMutationStartedAt ?? 0
763
+ const elapsedSeconds = Math.max(0, Math.round((Date.now() - startedAt) / 1000))
764
+ const holderPid = ownerMutationPid ?? ownerClientPid
765
+
766
+ return [
767
+ `owner mutation "${ownerMutationOperation || "unnamed operation"}"`,
768
+ `mutation ${ownerMutationId || "unknown"}`,
769
+ `held by process ${holderPid ?? "unknown"}`,
770
+ `since ${new Date(startedAt).toISOString()}`,
771
+ `${elapsedSeconds}s elapsed`,
772
+ `waiting for ${ownerMutationWaitingFor || "the operation to report completion"}`
773
+ ].join("; ")
774
+ }
775
+
776
+ /** @returns {void} */
777
+ function clearOwnerMutationLease() {
778
+ if (ownerMutationLeaseTimer) clearTimeout(ownerMutationLeaseTimer)
779
+ ownerMutationLeaseTimer = undefined
780
+ ownerMutationClient = undefined
781
+ ownerMutationId = undefined
782
+ ownerMutationOperation = undefined
783
+ ownerMutationWaitingFor = undefined
784
+ ownerMutationPid = undefined
785
+ ownerMutationStartedAt = undefined
786
+ ownerMutationLeaseMs = OWNER_MUTATION_LEASE_MS
787
+ }
788
+
789
+ /**
790
+ * Arms the lease bound for one mutation.
791
+ * @param {string} mutationId - Lease id the timer belongs to.
792
+ * @returns {void}
793
+ */
794
+ function armOwnerMutationLease(mutationId) {
795
+ if (ownerMutationLeaseTimer) clearTimeout(ownerMutationLeaseTimer)
796
+ ownerMutationLeaseTimer = setTimeout(() => expireOwnerMutationLease(mutationId), ownerMutationLeaseMs)
797
+ ownerMutationLeaseTimer.unref()
798
+ }
799
+
800
+ /**
801
+ * A mutation that outruns its lease is reported and released instead of holding the fence
802
+ * forever, so a stuck mutation cannot lock out every later deploy. The holder learns the
803
+ * lease is gone when it tries to end it.
804
+ * @param {string} mutationId - Lease id the armed timer belongs to.
805
+ * @returns {void}
806
+ */
807
+ function expireOwnerMutationLease(mutationId) {
808
+ ownerMutationLeaseTimer = undefined
809
+ if (!ownerMutationClient || ownerMutationId !== mutationId) return
810
+ const operation = ownerMutationOperation || "unnamed operation"
811
+ const leaseMs = ownerMutationLeaseMs
812
+ const elapsedMs = Math.max(0, Date.now() - (ownerMutationStartedAt ?? 0))
813
+ const diagnostic = {
814
+ code: "OWNER_MUTATION_LEASE_EXPIRED",
815
+ elapsedMs,
816
+ event: "owner-mutation-expired",
817
+ leaseMs,
818
+ mutationId,
819
+ operation,
820
+ pid: ownerMutationPid ?? ownerClientPid,
821
+ waitingFor: ownerMutationWaitingFor || "the operation to report completion"
822
+ }
823
+
824
+ process.stderr.write(`${JSON.stringify({at: new Date().toISOString(), ...diagnostic})}\n`)
825
+ if (!ownerMutationClient.destroyed) ownerMutationClient.write(`${JSON.stringify(diagnostic)}\n`)
826
+ broadcast(diagnostic)
827
+ ownerMutationExpired = {elapsedMs, leaseMs, mutationId, operation}
828
+ clearOwnerMutationLease()
829
+ }
830
+
718
831
  /**
719
832
  * @param {net.Socket} socket - Requesting client.
720
833
  * @param {string} command - Command name.
@@ -735,10 +848,7 @@ function releaseClosedOwner(socket) {
735
848
  rememberDisconnectedOwnerSuccessors(socket)
736
849
  ownerClient = undefined
737
850
  ownerClientPid = undefined
738
- if (ownerMutationClient === socket) {
739
- ownerMutationClient = undefined
740
- ownerMutationId = undefined
741
- }
851
+ if (ownerMutationClient === socket) clearOwnerMutationLease()
742
852
  if (replacementClient && replacementOwnerState) commitReplacement()
743
853
  else {
744
854
  if (incompleteRestartChild) {
@@ -58,6 +58,38 @@ test("guardian inventory removes only an exact owned provenance", async () => {
58
58
  }
59
59
  })
60
60
 
61
+ test("a refused owner mutation names the held mutation, its process, its age and its blocker", async () => {
62
+ const fixture = await createGuardian()
63
+
64
+ try {
65
+ const mutationId = await fixture.client.beginOwnerMutation("restart", {ownerPid: process.pid, waitingFor: "the managed process to stop and start again"})
66
+ await expect(fixture.client.beginOwnerMutation("deploy")).rejects.toThrow(/^Another owner mutation is already in progress: owner mutation "restart"; mutation [0-9a-f-]{36}; held by process \d+; since 20\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\.\d\d\dZ; \d+s elapsed; waiting for the managed process to stop and start again; refused deploy$/)
67
+ await fixture.client.reportOwnerMutationWait(mutationId, "an idle forked runner that never exits")
68
+ await expect(fixture.client.beginOwnerMutation("deploy")).rejects.toThrow(/waiting for an idle forked runner that never exits/)
69
+ await fixture.client.endOwnerMutation(mutationId)
70
+ expect(typeof await fixture.client.beginOwnerMutation("deploy")).toBe("string")
71
+ } finally {
72
+ await cleanupGuardian(fixture)
73
+ }
74
+ })
75
+
76
+ test("an owner mutation that outruns its lease is released and the holder reports the expiry", async () => {
77
+ const fixture = await createGuardian()
78
+
79
+ try {
80
+ const wedged = await fixture.client.beginOwnerMutation("restart", {mutationTimeoutMs: 200, ownerPid: process.pid, waitingFor: "a drain that never returns"})
81
+
82
+ await new Promise((resolve) => setTimeout(resolve, 600))
83
+ const successor = await fixture.client.beginOwnerMutation("deploy", {ownerPid: process.pid, waitingFor: "candidate activation"})
84
+
85
+ expect(typeof successor).toBe("string")
86
+ await expect(fixture.client.endOwnerMutation(wedged)).rejects.toThrow(/^Owner mutation restart exceeded its 200 ms lease and was released after \d+ ms; the mutation failed and its lease is gone$/)
87
+ await fixture.client.endOwnerMutation(successor)
88
+ } finally {
89
+ await cleanupGuardian(fixture)
90
+ }
91
+ })
92
+
61
93
  test("guardian lets the owner re-register a key whose registered child is gone, and still fences a live one", async () => {
62
94
  const fixture = await createGuardian()
63
95
  const liveInstance = fixture.client.process("live-generation", definition("live-generation"))