rollbridge 0.1.58 → 0.1.59
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
### Fixed
|
|
2
|
+
|
|
3
|
+
- A refused owner mutation now names its holder. The refusal used to read
|
|
4
|
+
`Another owner mutation is already in progress` with no context, so an
|
|
5
|
+
operator could not tell which command held the fence, which process held it,
|
|
6
|
+
for how long, or what it was blocked on. The guardian now records the
|
|
7
|
+
mutation's operation, lease id, holding PID, start time and current blocker
|
|
8
|
+
(`waitingFor`, refinable while the mutation runs through
|
|
9
|
+
`reportOwnerMutationWait`) and reports them in every refusal, including the
|
|
10
|
+
replacement-prepare fence.
|
|
11
|
+
|
|
12
|
+
### Changed
|
|
13
|
+
|
|
14
|
+
- The owner-mutation fence is bounded. A mutation that outruns its lease
|
|
15
|
+
(default 15 minutes, per-command override through `mutationTimeoutMs`) is
|
|
16
|
+
logged as `OWNER_MUTATION_LEASE_EXPIRED`, reported to the holder, and
|
|
17
|
+
released, so a stuck mutation can no longer lock out every later deploy. The
|
|
18
|
+
holder fails loudly when it next ends that mutation instead of silently
|
|
19
|
+
keeping the fence. Retained generation drains are unaffected: they are not
|
|
20
|
+
owner mutations and may still run for hours.
|
package/docs/troubleshooting.md
CHANGED
|
@@ -148,3 +148,27 @@ separate destructive `--takeover-owner` path. If
|
|
|
148
148
|
`retirementError` is set,
|
|
149
149
|
inspect the quiet-hook events. Rollbridge deliberately leaves that generation
|
|
150
150
|
alive rather than signaling arbitrary PIDs or continuing its stop sequence.
|
|
151
|
+
|
|
152
|
+
## A command is refused with "Another owner mutation is already in progress"
|
|
153
|
+
|
|
154
|
+
**Symptom.** `deploy`, `stop`, `restart`, `rollback` or
|
|
155
|
+
`recover-generation-transition` fails with
|
|
156
|
+
`Another owner mutation is already in progress: owner mutation "<operation>";
|
|
157
|
+
mutation <id>; held by process <pid>; since <ISO-8601>; <n>s elapsed; waiting
|
|
158
|
+
for <blocker>; refused <operation>`.
|
|
159
|
+
|
|
160
|
+
**Cause.** The committed daemon holds the guardian's owner-mutation fence for
|
|
161
|
+
one control mutation at a time, and a second mutation arrived while it was still
|
|
162
|
+
held.
|
|
163
|
+
|
|
164
|
+
**Fix.** Read the named holder before touching anything: the message states
|
|
165
|
+
which mutation holds the fence, its lease id, the PID of the process that
|
|
166
|
+
started it, when it started, how long it has run, and the blocker that holder
|
|
167
|
+
last reported (`waitingFor`, which a running mutation can refine through
|
|
168
|
+
`reportOwnerMutationWait`). That is the command to inspect — not the one that
|
|
169
|
+
was refused. A mutation that outruns its lease (15 minutes by default,
|
|
170
|
+
overridable per command with `mutationTimeoutMs`) is logged as
|
|
171
|
+
`OWNER_MUTATION_LEASE_EXPIRED`, reported to the holder, and released, so the
|
|
172
|
+
next mutation proceeds and the stuck one fails loudly when it next ends its
|
|
173
|
+
lease. A retained jobs generation is not an owner mutation and may still drain
|
|
174
|
+
for hours; only the fence is bounded.
|
package/package.json
CHANGED
package/src/daemon.js
CHANGED
|
@@ -146,7 +146,7 @@ export default class RollbridgeDaemon {
|
|
|
146
146
|
this.logger("release generation transition recovery failed", {error: error instanceof Error ? error.message : String(error), releaseId: transition.candidateReleaseId})
|
|
147
147
|
}
|
|
148
148
|
return {}
|
|
149
|
-
})
|
|
149
|
+
}, {waitingFor: "the retained release generations to settle"})
|
|
150
150
|
} finally {
|
|
151
151
|
this.generationTransitionRecovery = false
|
|
152
152
|
}
|
|
@@ -1389,7 +1389,7 @@ export default class RollbridgeDaemon {
|
|
|
1389
1389
|
releaseId: stringOrUndefined(data.releaseId),
|
|
1390
1390
|
releasePath: requiredString(data.releasePath, "releasePath"),
|
|
1391
1391
|
revision: stringOrUndefined(data.revision)
|
|
1392
|
-
}))
|
|
1392
|
+
}), {waitingFor: "candidate activation and the retired generation handoff"})
|
|
1393
1393
|
}
|
|
1394
1394
|
|
|
1395
1395
|
if (commandName === "recover-generation-transition") {
|
|
@@ -1399,7 +1399,7 @@ export default class RollbridgeDaemon {
|
|
|
1399
1399
|
releaseId: requiredString(data.releaseId, "releaseId"),
|
|
1400
1400
|
releasePath: requiredString(data.releasePath, "releasePath"),
|
|
1401
1401
|
revision: requiredString(data.revision, "revision")
|
|
1402
|
-
}))
|
|
1402
|
+
}), {waitingFor: "the retained release generations to settle"})
|
|
1403
1403
|
}
|
|
1404
1404
|
|
|
1405
1405
|
if (commandName === "status") {
|
|
@@ -1414,18 +1414,18 @@ export default class RollbridgeDaemon {
|
|
|
1414
1414
|
return await this.executeOwnerMutation("stop", async () => {
|
|
1415
1415
|
await this.stopRelease(stringOrUndefined(data.releaseId))
|
|
1416
1416
|
return this.status()
|
|
1417
|
-
})
|
|
1417
|
+
}, {waitingFor: "the release's managed processes to stop"})
|
|
1418
1418
|
}
|
|
1419
1419
|
|
|
1420
1420
|
if (commandName === "restart") {
|
|
1421
1421
|
return await this.executeOwnerMutation("restart", async () => await this.restartProcesses({
|
|
1422
1422
|
policy: stringOrUndefined(data.policy),
|
|
1423
1423
|
processId: stringOrUndefined(data.processId)
|
|
1424
|
-
}))
|
|
1424
|
+
}), {waitingFor: "the managed process to stop and start again"})
|
|
1425
1425
|
}
|
|
1426
1426
|
|
|
1427
1427
|
if (commandName === "rollback") {
|
|
1428
|
-
return await this.executeOwnerMutation("rollback", async () => await this.rollback({releaseId: stringOrUndefined(data.releaseId)}))
|
|
1428
|
+
return await this.executeOwnerMutation("rollback", async () => await this.rollback({releaseId: stringOrUndefined(data.releaseId)}), {waitingFor: "the previous release to activate"})
|
|
1429
1429
|
}
|
|
1430
1430
|
|
|
1431
1431
|
if (commandName === "shutdown") {
|
|
@@ -1489,11 +1489,13 @@ export default class RollbridgeDaemon {
|
|
|
1489
1489
|
* Runs one owner mutation under the guardian's prepare/commit exclusion fence.
|
|
1490
1490
|
* @param {string} operation - Mutation diagnostic name.
|
|
1491
1491
|
* @param {() => Promise<Record<string, JsonValue>>} callback - Mutating operation.
|
|
1492
|
+
* @param {{waitingFor?: string}} [options] - What this mutation blocks on, so a refused
|
|
1493
|
+
* later mutation names the real blocker instead of just "in progress".
|
|
1492
1494
|
* @returns {Promise<Record<string, JsonValue>>} Operation result.
|
|
1493
1495
|
*/
|
|
1494
|
-
async executeOwnerMutation(operation, callback) {
|
|
1496
|
+
async executeOwnerMutation(operation, callback, {waitingFor} = {}) {
|
|
1495
1497
|
if (!this.guardian) return await callback()
|
|
1496
|
-
const mutationId = await this.guardian.beginOwnerMutation(operation)
|
|
1498
|
+
const mutationId = await this.guardian.beginOwnerMutation(operation, {ownerPid: process.pid, waitingFor})
|
|
1497
1499
|
let result
|
|
1498
1500
|
let operationError
|
|
1499
1501
|
|
package/src/guardian-client.js
CHANGED
|
@@ -323,10 +323,11 @@ export default class GuardianClient {
|
|
|
323
323
|
/**
|
|
324
324
|
* Acquires the committed owner's mutation fence.
|
|
325
325
|
* @param {string} operation - Control mutation name.
|
|
326
|
+
* @param {{mutationTimeoutMs?: number, ownerPid?: number, waitingFor?: string}} [details] - Lease bound, holding PID and what the mutation blocks on.
|
|
326
327
|
* @returns {Promise<string>} Mutation lease id.
|
|
327
328
|
*/
|
|
328
|
-
async beginOwnerMutation(operation) {
|
|
329
|
-
const result = /** @type {{mutationId: string}} */ (await this.request({command: "begin-owner-mutation", operation}))
|
|
329
|
+
async beginOwnerMutation(operation, {mutationTimeoutMs, ownerPid, waitingFor} = {}) {
|
|
330
|
+
const result = /** @type {{mutationId: string}} */ (await this.request({command: "begin-owner-mutation", mutationTimeoutMs, operation, ownerPid, waitingFor}))
|
|
330
331
|
|
|
331
332
|
return result.mutationId
|
|
332
333
|
}
|
|
@@ -336,6 +337,16 @@ export default class GuardianClient {
|
|
|
336
337
|
await this.request({command: "end-owner-mutation", mutationId})
|
|
337
338
|
}
|
|
338
339
|
|
|
340
|
+
/**
|
|
341
|
+
* Refines what a held mutation is waiting for, so a later refusal names the real blocker.
|
|
342
|
+
* @param {string} mutationId - Mutation lease id.
|
|
343
|
+
* @param {string} waitingFor - Current blocker.
|
|
344
|
+
* @returns {Promise<void>} Report acknowledged.
|
|
345
|
+
*/
|
|
346
|
+
async reportOwnerMutationWait(mutationId, waitingFor) {
|
|
347
|
+
await this.request({command: "report-owner-mutation-wait", mutationId, waitingFor})
|
|
348
|
+
}
|
|
349
|
+
|
|
339
350
|
/** @returns {Promise<{committedReplacementId: string | null, ownerClaimed: boolean, retirementFailed?: boolean, retirementPending?: boolean, retirementReady?: boolean}>} Transaction status. */
|
|
340
351
|
async replacementStatus() {
|
|
341
352
|
return /** @type {{committedReplacementId: string | null, ownerClaimed: boolean, retirementFailed?: boolean, retirementPending?: boolean, retirementReady?: boolean}} */ (await this.request({command: "replacement-status"}))
|
package/src/process-guardian.js
CHANGED
|
@@ -12,6 +12,10 @@ import ManagedProcess from "./managed-process.js"
|
|
|
12
12
|
import {writeState} from "./state-store.js"
|
|
13
13
|
|
|
14
14
|
const OWNER_RESTART_RETRY_MS = 1000
|
|
15
|
+
// An owner mutation is deploy/stop/restart/rollback/recovery work. A retained
|
|
16
|
+
// generation may legitimately drain for hours, but a mutation that holds the
|
|
17
|
+
// fence that long is stuck, so the lease is bounded and released on expiry.
|
|
18
|
+
const OWNER_MUTATION_LEASE_MS = 900000
|
|
15
19
|
|
|
16
20
|
/** @typedef {import("node:child_process").ChildProcess["signalCode"]} ChildExitSignal */
|
|
17
21
|
|
|
@@ -32,7 +36,9 @@ const OWNER_RESTART_RETRY_MS = 1000
|
|
|
32
36
|
* @property {import("./json.js").JsonValue} [ownerState] - Private owner transfer state.
|
|
33
37
|
* @property {string} [replacementId] - Prepared replacement transaction id.
|
|
34
38
|
* @property {string} [mutationId] - Owner mutation lease id.
|
|
39
|
+
* @property {number} [mutationTimeoutMs] - Bound for the owner mutation lease.
|
|
35
40
|
* @property {string} [operation] - Owner mutation diagnostic name.
|
|
41
|
+
* @property {string} [waitingFor] - What the held owner mutation is blocked on.
|
|
36
42
|
* @property {number} [ownerPid] - Exact claimant process PID.
|
|
37
43
|
* @property {import("./json.js").JsonValue} [recoverySnapshot] - Public bridge recovery snapshot.
|
|
38
44
|
* @property {string} [statePath] - Exact public recovery state path.
|
|
@@ -103,6 +109,19 @@ const legacyKeys = legacyGuardian ? committedOwnerProcessKeys(ownerState) : new
|
|
|
103
109
|
let ownerMutationClient
|
|
104
110
|
/** @type {string | undefined} */
|
|
105
111
|
let ownerMutationId
|
|
112
|
+
/** @type {string | undefined} */
|
|
113
|
+
let ownerMutationOperation
|
|
114
|
+
/** @type {string | undefined} */
|
|
115
|
+
let ownerMutationWaitingFor
|
|
116
|
+
/** @type {number | undefined} */
|
|
117
|
+
let ownerMutationPid
|
|
118
|
+
/** @type {number | undefined} */
|
|
119
|
+
let ownerMutationStartedAt
|
|
120
|
+
let ownerMutationLeaseMs = OWNER_MUTATION_LEASE_MS
|
|
121
|
+
/** @type {ReturnType<typeof setTimeout> | undefined} */
|
|
122
|
+
let ownerMutationLeaseTimer
|
|
123
|
+
/** @type {{elapsedMs: number, leaseMs: number, mutationId: string, operation: string} | undefined} */
|
|
124
|
+
let ownerMutationExpired
|
|
106
125
|
/** @type {net.Socket | undefined} */
|
|
107
126
|
let retiringClient
|
|
108
127
|
/** @type {string | undefined} */
|
|
@@ -154,10 +173,7 @@ const server = net.createServer((socket) => {
|
|
|
154
173
|
clearTimeout(waiter.timer)
|
|
155
174
|
waiter.reject(new Error("Owner claimant disconnected"))
|
|
156
175
|
}
|
|
157
|
-
if (ownerMutationClient === socket && ownerClient !== socket)
|
|
158
|
-
ownerMutationClient = undefined
|
|
159
|
-
ownerMutationId = undefined
|
|
160
|
-
}
|
|
176
|
+
if (ownerMutationClient === socket && ownerClient !== socket) clearOwnerMutationLease()
|
|
161
177
|
if (controlLessRetirement) {
|
|
162
178
|
const preCommitHandoff = replacementClient && ownerClient === socket && retiringReplacementId === replacementId
|
|
163
179
|
|
|
@@ -301,8 +317,7 @@ async function execute(request, socket) {
|
|
|
301
317
|
rememberDisconnectedOwnerSuccessors(socket)
|
|
302
318
|
ownerClient = undefined
|
|
303
319
|
ownerClientPid = undefined
|
|
304
|
-
|
|
305
|
-
ownerMutationId = undefined
|
|
320
|
+
clearOwnerMutationLease()
|
|
306
321
|
ownerRevision += 1
|
|
307
322
|
grantNextOwner()
|
|
308
323
|
return {retired: true}
|
|
@@ -397,26 +412,48 @@ async function execute(request, socket) {
|
|
|
397
412
|
if (request.command === "begin-owner-mutation") {
|
|
398
413
|
requireOwner(socket, request.command)
|
|
399
414
|
if (replacementClient) throw new Error(`Owner mutation ${request.operation || "operation"} is fenced while an owner replacement is prepared`)
|
|
400
|
-
|
|
415
|
+
const leaseMs = request.mutationTimeoutMs === undefined ? OWNER_MUTATION_LEASE_MS : request.mutationTimeoutMs
|
|
416
|
+
|
|
417
|
+
if (!Number.isInteger(leaseMs) || leaseMs <= 0) throw new Error("Owner mutation lease bound must be a positive number of milliseconds")
|
|
418
|
+
if (ownerMutationClient) throw new Error(`Another owner mutation is already in progress: ${describeOwnerMutationLease()}; refused ${request.operation || "operation"}`)
|
|
401
419
|
ownerMutationClient = socket
|
|
402
420
|
ownerMutationId = crypto.randomUUID()
|
|
403
|
-
|
|
421
|
+
ownerMutationOperation = request.operation
|
|
422
|
+
ownerMutationWaitingFor = request.waitingFor
|
|
423
|
+
ownerMutationPid = request.ownerPid ?? ownerClientPid
|
|
424
|
+
ownerMutationStartedAt = Date.now()
|
|
425
|
+
ownerMutationLeaseMs = leaseMs
|
|
426
|
+
armOwnerMutationLease(ownerMutationId)
|
|
427
|
+
return {leaseMs, mutationId: ownerMutationId}
|
|
404
428
|
}
|
|
405
429
|
|
|
406
430
|
if (request.command === "end-owner-mutation") {
|
|
407
431
|
requireOwner(socket, request.command)
|
|
432
|
+
if (ownerMutationExpired && request.mutationId === ownerMutationExpired.mutationId) {
|
|
433
|
+
const expired = ownerMutationExpired
|
|
434
|
+
|
|
435
|
+
ownerMutationExpired = undefined
|
|
436
|
+
throw new Error(`Owner mutation ${expired.operation} exceeded its ${expired.leaseMs} ms lease and was released after ${expired.elapsedMs} ms; the mutation failed and its lease is gone`)
|
|
437
|
+
}
|
|
408
438
|
if (ownerMutationClient !== socket || request.mutationId !== ownerMutationId) throw new Error("Owner mutation lease does not match the active mutation")
|
|
409
|
-
|
|
410
|
-
ownerMutationId = undefined
|
|
439
|
+
clearOwnerMutationLease()
|
|
411
440
|
return {ended: true}
|
|
412
441
|
}
|
|
413
442
|
|
|
443
|
+
if (request.command === "report-owner-mutation-wait") {
|
|
444
|
+
requireOwner(socket, request.command)
|
|
445
|
+
if (ownerMutationClient !== socket || request.mutationId !== ownerMutationId) throw new Error("Owner mutation lease does not match the active mutation")
|
|
446
|
+
if (typeof request.waitingFor !== "string" || !request.waitingFor.trim()) throw new Error("Owner mutation wait report requires a waitingFor description")
|
|
447
|
+
ownerMutationWaitingFor = request.waitingFor.trim()
|
|
448
|
+
return {reported: true}
|
|
449
|
+
}
|
|
450
|
+
|
|
414
451
|
if (request.command === "prepare-owner-replacement") {
|
|
415
452
|
if (retirementFailed) throw new Error("Guardian owner replacement is fenced after incomplete retired-listener state transfer")
|
|
416
453
|
if (retiringClient) throw new Error("Guardian owner replacement cannot prepare while listener retirement is pending")
|
|
417
454
|
if (socket === ownerClient) throw new Error("Committed owner cannot prepare itself as a replacement")
|
|
418
455
|
if (replacementClient && replacementClient !== socket) throw new Error("Another owner replacement candidate is already prepared")
|
|
419
|
-
if (ownerMutationClient) throw new Error(
|
|
456
|
+
if (ownerMutationClient) throw new Error(`Owner replacement cannot prepare while an owner mutation is in progress: ${describeOwnerMutationLease()}`)
|
|
420
457
|
if (!ownerState) throw new Error("Committed owner has not published transferable state")
|
|
421
458
|
const currentAuthority = ownerAuthority(ownerState)
|
|
422
459
|
const resumesCommittedAuthority = !ownerClient && isDeepStrictEqual(currentAuthority, request.nextAuthority)
|
|
@@ -715,6 +752,82 @@ async function execute(request, socket) {
|
|
|
715
752
|
return status
|
|
716
753
|
}
|
|
717
754
|
|
|
755
|
+
/**
|
|
756
|
+
* Names the mutation holding the owner-mutation fence: which mutation, which process,
|
|
757
|
+
* since when, and what it is blocked on. Refusals must be diagnosable without a live
|
|
758
|
+
* inspection of the daemon.
|
|
759
|
+
* @returns {string} Holder description used in refusal messages.
|
|
760
|
+
*/
|
|
761
|
+
function describeOwnerMutationLease() {
|
|
762
|
+
const startedAt = ownerMutationStartedAt ?? 0
|
|
763
|
+
const elapsedSeconds = Math.max(0, Math.round((Date.now() - startedAt) / 1000))
|
|
764
|
+
const holderPid = ownerMutationPid ?? ownerClientPid
|
|
765
|
+
|
|
766
|
+
return [
|
|
767
|
+
`owner mutation "${ownerMutationOperation || "unnamed operation"}"`,
|
|
768
|
+
`mutation ${ownerMutationId || "unknown"}`,
|
|
769
|
+
`held by process ${holderPid ?? "unknown"}`,
|
|
770
|
+
`since ${new Date(startedAt).toISOString()}`,
|
|
771
|
+
`${elapsedSeconds}s elapsed`,
|
|
772
|
+
`waiting for ${ownerMutationWaitingFor || "the operation to report completion"}`
|
|
773
|
+
].join("; ")
|
|
774
|
+
}
|
|
775
|
+
|
|
776
|
+
/** @returns {void} */
|
|
777
|
+
function clearOwnerMutationLease() {
|
|
778
|
+
if (ownerMutationLeaseTimer) clearTimeout(ownerMutationLeaseTimer)
|
|
779
|
+
ownerMutationLeaseTimer = undefined
|
|
780
|
+
ownerMutationClient = undefined
|
|
781
|
+
ownerMutationId = undefined
|
|
782
|
+
ownerMutationOperation = undefined
|
|
783
|
+
ownerMutationWaitingFor = undefined
|
|
784
|
+
ownerMutationPid = undefined
|
|
785
|
+
ownerMutationStartedAt = undefined
|
|
786
|
+
ownerMutationLeaseMs = OWNER_MUTATION_LEASE_MS
|
|
787
|
+
}
|
|
788
|
+
|
|
789
|
+
/**
|
|
790
|
+
* Arms the lease bound for one mutation.
|
|
791
|
+
* @param {string} mutationId - Lease id the timer belongs to.
|
|
792
|
+
* @returns {void}
|
|
793
|
+
*/
|
|
794
|
+
function armOwnerMutationLease(mutationId) {
|
|
795
|
+
if (ownerMutationLeaseTimer) clearTimeout(ownerMutationLeaseTimer)
|
|
796
|
+
ownerMutationLeaseTimer = setTimeout(() => expireOwnerMutationLease(mutationId), ownerMutationLeaseMs)
|
|
797
|
+
ownerMutationLeaseTimer.unref()
|
|
798
|
+
}
|
|
799
|
+
|
|
800
|
+
/**
|
|
801
|
+
* A mutation that outruns its lease is reported and released instead of holding the fence
|
|
802
|
+
* forever, so a stuck mutation cannot lock out every later deploy. The holder learns the
|
|
803
|
+
* lease is gone when it tries to end it.
|
|
804
|
+
* @param {string} mutationId - Lease id the armed timer belongs to.
|
|
805
|
+
* @returns {void}
|
|
806
|
+
*/
|
|
807
|
+
function expireOwnerMutationLease(mutationId) {
|
|
808
|
+
ownerMutationLeaseTimer = undefined
|
|
809
|
+
if (!ownerMutationClient || ownerMutationId !== mutationId) return
|
|
810
|
+
const operation = ownerMutationOperation || "unnamed operation"
|
|
811
|
+
const leaseMs = ownerMutationLeaseMs
|
|
812
|
+
const elapsedMs = Math.max(0, Date.now() - (ownerMutationStartedAt ?? 0))
|
|
813
|
+
const diagnostic = {
|
|
814
|
+
code: "OWNER_MUTATION_LEASE_EXPIRED",
|
|
815
|
+
elapsedMs,
|
|
816
|
+
event: "owner-mutation-expired",
|
|
817
|
+
leaseMs,
|
|
818
|
+
mutationId,
|
|
819
|
+
operation,
|
|
820
|
+
pid: ownerMutationPid ?? ownerClientPid,
|
|
821
|
+
waitingFor: ownerMutationWaitingFor || "the operation to report completion"
|
|
822
|
+
}
|
|
823
|
+
|
|
824
|
+
process.stderr.write(`${JSON.stringify({at: new Date().toISOString(), ...diagnostic})}\n`)
|
|
825
|
+
if (!ownerMutationClient.destroyed) ownerMutationClient.write(`${JSON.stringify(diagnostic)}\n`)
|
|
826
|
+
broadcast(diagnostic)
|
|
827
|
+
ownerMutationExpired = {elapsedMs, leaseMs, mutationId, operation}
|
|
828
|
+
clearOwnerMutationLease()
|
|
829
|
+
}
|
|
830
|
+
|
|
718
831
|
/**
|
|
719
832
|
* @param {net.Socket} socket - Requesting client.
|
|
720
833
|
* @param {string} command - Command name.
|
|
@@ -735,10 +848,7 @@ function releaseClosedOwner(socket) {
|
|
|
735
848
|
rememberDisconnectedOwnerSuccessors(socket)
|
|
736
849
|
ownerClient = undefined
|
|
737
850
|
ownerClientPid = undefined
|
|
738
|
-
if (ownerMutationClient === socket)
|
|
739
|
-
ownerMutationClient = undefined
|
|
740
|
-
ownerMutationId = undefined
|
|
741
|
-
}
|
|
851
|
+
if (ownerMutationClient === socket) clearOwnerMutationLease()
|
|
742
852
|
if (replacementClient && replacementOwnerState) commitReplacement()
|
|
743
853
|
else {
|
|
744
854
|
if (incompleteRestartChild) {
|
|
@@ -58,6 +58,38 @@ test("guardian inventory removes only an exact owned provenance", async () => {
|
|
|
58
58
|
}
|
|
59
59
|
})
|
|
60
60
|
|
|
61
|
+
test("a refused owner mutation names the held mutation, its process, its age and its blocker", async () => {
|
|
62
|
+
const fixture = await createGuardian()
|
|
63
|
+
|
|
64
|
+
try {
|
|
65
|
+
const mutationId = await fixture.client.beginOwnerMutation("restart", {ownerPid: process.pid, waitingFor: "the managed process to stop and start again"})
|
|
66
|
+
await expect(fixture.client.beginOwnerMutation("deploy")).rejects.toThrow(/^Another owner mutation is already in progress: owner mutation "restart"; mutation [0-9a-f-]{36}; held by process \d+; since 20\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\.\d\d\dZ; \d+s elapsed; waiting for the managed process to stop and start again; refused deploy$/)
|
|
67
|
+
await fixture.client.reportOwnerMutationWait(mutationId, "an idle forked runner that never exits")
|
|
68
|
+
await expect(fixture.client.beginOwnerMutation("deploy")).rejects.toThrow(/waiting for an idle forked runner that never exits/)
|
|
69
|
+
await fixture.client.endOwnerMutation(mutationId)
|
|
70
|
+
expect(typeof await fixture.client.beginOwnerMutation("deploy")).toBe("string")
|
|
71
|
+
} finally {
|
|
72
|
+
await cleanupGuardian(fixture)
|
|
73
|
+
}
|
|
74
|
+
})
|
|
75
|
+
|
|
76
|
+
test("an owner mutation that outruns its lease is released and the holder reports the expiry", async () => {
|
|
77
|
+
const fixture = await createGuardian()
|
|
78
|
+
|
|
79
|
+
try {
|
|
80
|
+
const wedged = await fixture.client.beginOwnerMutation("restart", {mutationTimeoutMs: 200, ownerPid: process.pid, waitingFor: "a drain that never returns"})
|
|
81
|
+
|
|
82
|
+
await new Promise((resolve) => setTimeout(resolve, 600))
|
|
83
|
+
const successor = await fixture.client.beginOwnerMutation("deploy", {ownerPid: process.pid, waitingFor: "candidate activation"})
|
|
84
|
+
|
|
85
|
+
expect(typeof successor).toBe("string")
|
|
86
|
+
await expect(fixture.client.endOwnerMutation(wedged)).rejects.toThrow(/^Owner mutation restart exceeded its 200 ms lease and was released after \d+ ms; the mutation failed and its lease is gone$/)
|
|
87
|
+
await fixture.client.endOwnerMutation(successor)
|
|
88
|
+
} finally {
|
|
89
|
+
await cleanupGuardian(fixture)
|
|
90
|
+
}
|
|
91
|
+
})
|
|
92
|
+
|
|
61
93
|
test("guardian lets the owner re-register a key whose registered child is gone, and still fences a live one", async () => {
|
|
62
94
|
const fixture = await createGuardian()
|
|
63
95
|
const liveInstance = fixture.client.process("live-generation", definition("live-generation"))
|