rollbridge 0.1.73 → 0.1.74
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +63 -10
- package/TODO.md +13 -0
- package/changelog.d/20261025-guardian-re-register-dead-generation.md +13 -0
- package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +1 -0
- package/changelog.d/20261104-compose-service-units.md +21 -0
- package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +20 -0
- package/changelog.d/20261106-release-dependency-boot-budgets.md +32 -0
- package/changelog.d/20261109-path-filtered-builds.md +11 -0
- package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +26 -0
- package/changelog.d/20261111-contention-sensitive-daemon-specs.md +27 -0
- package/changelog.d/20261113-bounded-nonblocking-retirement.md +5 -0
- package/changelog.d/20261115-bound-draining-release-groups.md +62 -0
- package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +27 -0
- package/changelog.d/20261115-wedged-generation-escalation.md +44 -0
- package/changelog.d/20261116-bounded-explicit-stop-retirement.md +30 -0
- package/changelog.d/20261116-config-json-canonical-shape.md +11 -0
- package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +29 -0
- package/changelog.d/20261116-guardian-version-handoff.md +37 -0
- package/changelog.d/20261116-reclaim-dead-owner-mutation.md +7 -0
- package/changelog.d/20261117-activate-before-incumbent-retirement.md +1 -0
- package/changelog.d/20261117-activation-path-drain-deadline.md +35 -0
- package/changelog.d/20261117-stop-companion-after-retired-dependency.md +7 -0
- package/changelog.d/20261118-incumbent-retirement-authority-guard.md +1 -0
- package/changelog.d/20261118-owner-retirement-no-guardian-bound.md +19 -0
- package/changelog.d/20261118-shutdown-retirement-bound-restored.md +31 -0
- package/docs/cli.md +128 -3
- package/docs/compose-services.md +222 -0
- package/docs/config.md +177 -6
- package/docs/troubleshooting.md +97 -6
- package/docs/velocious.md +1 -1
- package/docs/workers.md +11 -0
- package/examples/tensorbuzz.com.js +3 -2
- package/package.json +1 -1
- package/src/cli.js +68 -2
- package/src/compose-unit.js +725 -0
- package/src/config.js +346 -8
- package/src/daemon.js +411 -78
- package/src/doctor.js +74 -0
- package/src/drain-policy.js +252 -0
- package/src/guardian-client.js +68 -23
- package/src/health.js +99 -8
- package/src/managed-process.js +198 -45
- package/src/process-guardian.js +212 -37
- package/src/process-memory.js +58 -23
- package/src/process-order.js +26 -0
- package/src/release-group.js +438 -45
- package/tensorbuzz.yml +20 -1
- package/test/completion.test.js +5 -2
- package/test/compose-config.test.js +160 -0
- package/test/compose-docker.test.js +297 -0
- package/test/compose-unit.test.js +415 -0
- package/test/config-validation.test.js +135 -5
- package/test/drain-policy.test.js +328 -0
- package/test/fixtures/deferred-boot-app.js +106 -0
- package/test/fixtures/gated-app.js +50 -0
- package/test/guardian-client.test.js +119 -5
- package/test/health.test.js +74 -1
- package/test/managed-process.test.js +260 -24
- package/test/owner-recovery.test.js +125 -21
- package/test/owner-replacement.test.js +175 -14
- package/test/process-memory.test.js +47 -1
- package/test/release-group.test.js +408 -2
- package/test/rollbridge.test.js +774 -72
- package/test/shutdown-completion.test.js +119 -0
- package/test/shutdown-retirement-bound.test.js +162 -0
- package/test/support/process.js +3 -1
- package/test/tensorbuzz-config.test.js +228 -0
package/src/process-guardian.js
CHANGED
|
@@ -4,14 +4,21 @@ import fsSync from "node:fs"
|
|
|
4
4
|
import fs from "node:fs/promises"
|
|
5
5
|
import net from "node:net"
|
|
6
6
|
import crypto from "node:crypto"
|
|
7
|
+
import {createRequire} from "node:module"
|
|
7
8
|
import {spawn} from "node:child_process"
|
|
8
9
|
import {isDeepStrictEqual} from "node:util"
|
|
9
10
|
import path from "node:path"
|
|
10
11
|
import GuardianClient from "./guardian-client.js"
|
|
11
|
-
import ManagedProcess from "./managed-process.js"
|
|
12
|
+
import ManagedProcess, {childRecordIsGone} from "./managed-process.js"
|
|
12
13
|
import {writeState} from "./state-store.js"
|
|
13
14
|
|
|
14
15
|
const OWNER_RESTART_RETRY_MS = 1000
|
|
16
|
+
const commonJsRequire = createRequire(import.meta.url)
|
|
17
|
+
const GUARDIAN_VERSION = /** @type {{version: string}} */ (commonJsRequire("../package.json")).version
|
|
18
|
+
// An owner mutation is deploy/stop/restart/rollback/recovery work. A retained
|
|
19
|
+
// generation may legitimately drain for hours, but a mutation that holds the
|
|
20
|
+
// fence that long is stuck, so the lease is bounded and released on expiry.
|
|
21
|
+
const OWNER_MUTATION_LEASE_MS = 900000
|
|
15
22
|
|
|
16
23
|
/** @typedef {import("node:child_process").ChildProcess["signalCode"]} ChildExitSignal */
|
|
17
24
|
|
|
@@ -24,7 +31,7 @@ const OWNER_RESTART_RETRY_MS = 1000
|
|
|
24
31
|
* @property {number} id - Request id.
|
|
25
32
|
* @property {string} [key] - Stable process key.
|
|
26
33
|
* @property {boolean} [localSource] - Whether the retired daemon physically owns the published source.
|
|
27
|
-
* @property {{timeoutMs?: number}} [options] - Stop options.
|
|
34
|
+
* @property {{retirementTimeoutMs?: number, timeoutMs?: number}} [options] - Stop options.
|
|
28
35
|
* @property {string} [previousProvenance] - Expected current provenance.
|
|
29
36
|
* @property {string} [provenance] - New or registered provenance.
|
|
30
37
|
* @property {string} [releaseId] - Retained release identity.
|
|
@@ -32,11 +39,10 @@ const OWNER_RESTART_RETRY_MS = 1000
|
|
|
32
39
|
* @property {import("./json.js").JsonValue} [ownerState] - Private owner transfer state.
|
|
33
40
|
* @property {string} [replacementId] - Prepared replacement transaction id.
|
|
34
41
|
* @property {string} [mutationId] - Owner mutation lease id.
|
|
42
|
+
* @property {number} [mutationTimeoutMs] - Bound for the owner mutation lease.
|
|
35
43
|
* @property {string} [operation] - Owner mutation diagnostic name.
|
|
44
|
+
* @property {string} [waitingFor] - What the held owner mutation is blocked on.
|
|
36
45
|
* @property {number} [ownerPid] - Exact claimant process PID.
|
|
37
|
-
* @property {string} [configDigest] - Exact committed owner configuration digest.
|
|
38
|
-
* @property {number} [pid] - Exact live process PID to re-adopt.
|
|
39
|
-
* @property {string} [processStartTime] - Exact `/proc` process start-time counter.
|
|
40
46
|
* @property {import("./json.js").JsonValue} [recoverySnapshot] - Public bridge recovery snapshot.
|
|
41
47
|
* @property {string} [statePath] - Exact public recovery state path.
|
|
42
48
|
* @property {import("./json.js").JsonValue} [authority] - Expected current authority.
|
|
@@ -106,6 +112,19 @@ const legacyKeys = legacyGuardian ? committedOwnerProcessKeys(ownerState) : new
|
|
|
106
112
|
let ownerMutationClient
|
|
107
113
|
/** @type {string | undefined} */
|
|
108
114
|
let ownerMutationId
|
|
115
|
+
/** @type {string | undefined} */
|
|
116
|
+
let ownerMutationOperation
|
|
117
|
+
/** @type {string | undefined} */
|
|
118
|
+
let ownerMutationWaitingFor
|
|
119
|
+
/** @type {number | undefined} */
|
|
120
|
+
let ownerMutationPid
|
|
121
|
+
/** @type {number | undefined} */
|
|
122
|
+
let ownerMutationStartedAt
|
|
123
|
+
let ownerMutationLeaseMs = OWNER_MUTATION_LEASE_MS
|
|
124
|
+
/** @type {ReturnType<typeof setTimeout> | undefined} */
|
|
125
|
+
let ownerMutationLeaseTimer
|
|
126
|
+
/** @type {{elapsedMs: number, leaseMs: number, mutationId: string, operation: string} | undefined} */
|
|
127
|
+
let ownerMutationExpired
|
|
109
128
|
/** @type {net.Socket | undefined} */
|
|
110
129
|
let retiringClient
|
|
111
130
|
/** @type {string | undefined} */
|
|
@@ -157,10 +176,7 @@ const server = net.createServer((socket) => {
|
|
|
157
176
|
clearTimeout(waiter.timer)
|
|
158
177
|
waiter.reject(new Error("Owner claimant disconnected"))
|
|
159
178
|
}
|
|
160
|
-
if (ownerMutationClient === socket
|
|
161
|
-
ownerMutationClient = undefined
|
|
162
|
-
ownerMutationId = undefined
|
|
163
|
-
}
|
|
179
|
+
if (ownerMutationClient === socket) clearOwnerMutationLease()
|
|
164
180
|
if (controlLessRetirement) {
|
|
165
181
|
const preCommitHandoff = replacementClient && ownerClient === socket && retiringReplacementId === replacementId
|
|
166
182
|
|
|
@@ -262,13 +278,14 @@ async function handleLine(socket, line) {
|
|
|
262
278
|
async function execute(request, socket) {
|
|
263
279
|
if (shuttingDown) throw new Error("Process guardian is shutting down")
|
|
264
280
|
|
|
265
|
-
if (request.command === "capabilities") return {daemonRecovery: 1, generationReactivation: 1}
|
|
281
|
+
if (request.command === "capabilities") return {daemonRecovery: 1, generationReactivation: 1, guardianVersion: GUARDIAN_VERSION}
|
|
266
282
|
|
|
267
283
|
if (request.command === "owner-replacement-capabilities") {
|
|
268
284
|
return {commands: ["commit-retired-owner-replacement"], protocol: "owner-replacement", version: 1}
|
|
269
285
|
}
|
|
270
286
|
|
|
271
287
|
if (request.command === "claim-owner") {
|
|
288
|
+
reclaimDeadOwnerMutation()
|
|
272
289
|
if (retirementFailed) throw new Error("Guardian owner replacement is fenced after incomplete retired-listener state transfer")
|
|
273
290
|
if (ownerState !== undefined && !isDeepStrictEqual(request.authority, ownerAuthority(ownerState))) {
|
|
274
291
|
const waitsForPreparedAuthority = Boolean(ownerClient && replacementOwnerState && isDeepStrictEqual(request.authority, replacementAuthority))
|
|
@@ -304,8 +321,7 @@ async function execute(request, socket) {
|
|
|
304
321
|
rememberDisconnectedOwnerSuccessors(socket)
|
|
305
322
|
ownerClient = undefined
|
|
306
323
|
ownerClientPid = undefined
|
|
307
|
-
|
|
308
|
-
ownerMutationId = undefined
|
|
324
|
+
clearOwnerMutationLease()
|
|
309
325
|
ownerRevision += 1
|
|
310
326
|
grantNextOwner()
|
|
311
327
|
return {retired: true}
|
|
@@ -397,29 +413,54 @@ async function execute(request, socket) {
|
|
|
397
413
|
return {committedReplacementId: committedReplacementId ?? null, ownerClaimed: Boolean(ownerClient), retirementFailed, retirementPending: Boolean(retiringClient), retirementReady: retiringListenerReady}
|
|
398
414
|
}
|
|
399
415
|
|
|
416
|
+
if (request.command === "guardian-status") return guardianStatus()
|
|
417
|
+
|
|
400
418
|
if (request.command === "begin-owner-mutation") {
|
|
401
419
|
requireOwner(socket, request.command)
|
|
420
|
+
reclaimDeadOwnerMutation()
|
|
402
421
|
if (replacementClient) throw new Error(`Owner mutation ${request.operation || "operation"} is fenced while an owner replacement is prepared`)
|
|
403
|
-
|
|
422
|
+
const leaseMs = request.mutationTimeoutMs === undefined ? OWNER_MUTATION_LEASE_MS : request.mutationTimeoutMs
|
|
423
|
+
|
|
424
|
+
if (!Number.isInteger(leaseMs) || leaseMs <= 0) throw new Error("Owner mutation lease bound must be a positive number of milliseconds")
|
|
425
|
+
if (ownerMutationClient) throw new Error(`Another owner mutation is already in progress: ${describeOwnerMutationLease()}; refused ${request.operation || "operation"}`)
|
|
404
426
|
ownerMutationClient = socket
|
|
405
427
|
ownerMutationId = crypto.randomUUID()
|
|
406
|
-
|
|
428
|
+
ownerMutationOperation = request.operation
|
|
429
|
+
ownerMutationWaitingFor = request.waitingFor
|
|
430
|
+
ownerMutationPid = request.ownerPid ?? ownerClientPid
|
|
431
|
+
ownerMutationStartedAt = Date.now()
|
|
432
|
+
ownerMutationLeaseMs = leaseMs
|
|
433
|
+
armOwnerMutationLease(ownerMutationId)
|
|
434
|
+
return {leaseMs, mutationId: ownerMutationId}
|
|
407
435
|
}
|
|
408
436
|
|
|
409
437
|
if (request.command === "end-owner-mutation") {
|
|
410
438
|
requireOwner(socket, request.command)
|
|
439
|
+
if (ownerMutationExpired && request.mutationId === ownerMutationExpired.mutationId) {
|
|
440
|
+
const expired = ownerMutationExpired
|
|
441
|
+
|
|
442
|
+
ownerMutationExpired = undefined
|
|
443
|
+
throw new Error(`Owner mutation ${expired.operation} exceeded its ${expired.leaseMs} ms lease and was released after ${expired.elapsedMs} ms; the mutation failed and its lease is gone`)
|
|
444
|
+
}
|
|
411
445
|
if (ownerMutationClient !== socket || request.mutationId !== ownerMutationId) throw new Error("Owner mutation lease does not match the active mutation")
|
|
412
|
-
|
|
413
|
-
ownerMutationId = undefined
|
|
446
|
+
clearOwnerMutationLease()
|
|
414
447
|
return {ended: true}
|
|
415
448
|
}
|
|
416
449
|
|
|
450
|
+
if (request.command === "report-owner-mutation-wait") {
|
|
451
|
+
requireOwner(socket, request.command)
|
|
452
|
+
if (ownerMutationClient !== socket || request.mutationId !== ownerMutationId) throw new Error("Owner mutation lease does not match the active mutation")
|
|
453
|
+
if (typeof request.waitingFor !== "string" || !request.waitingFor.trim()) throw new Error("Owner mutation wait report requires a waitingFor description")
|
|
454
|
+
ownerMutationWaitingFor = request.waitingFor.trim()
|
|
455
|
+
return {reported: true}
|
|
456
|
+
}
|
|
457
|
+
|
|
417
458
|
if (request.command === "prepare-owner-replacement") {
|
|
418
459
|
if (retirementFailed) throw new Error("Guardian owner replacement is fenced after incomplete retired-listener state transfer")
|
|
419
460
|
if (retiringClient) throw new Error("Guardian owner replacement cannot prepare while listener retirement is pending")
|
|
420
461
|
if (socket === ownerClient) throw new Error("Committed owner cannot prepare itself as a replacement")
|
|
421
462
|
if (replacementClient && replacementClient !== socket) throw new Error("Another owner replacement candidate is already prepared")
|
|
422
|
-
if (ownerMutationClient) throw new Error(
|
|
463
|
+
if (ownerMutationClient) throw new Error(`Owner replacement cannot prepare while an owner mutation is in progress: ${describeOwnerMutationLease()}`)
|
|
423
464
|
if (!ownerState) throw new Error("Committed owner has not published transferable state")
|
|
424
465
|
const currentAuthority = ownerAuthority(ownerState)
|
|
425
466
|
const resumesCommittedAuthority = !ownerClient && isDeepStrictEqual(currentAuthority, request.nextAuthority)
|
|
@@ -448,7 +489,7 @@ async function execute(request, socket) {
|
|
|
448
489
|
|
|
449
490
|
if (request.command === "stage-owner-replacement") {
|
|
450
491
|
requireReplacement(socket, request)
|
|
451
|
-
if (replacementRevision !== ownerRevision) throw new Error(
|
|
492
|
+
if (replacementRevision !== ownerRevision) throw new Error(`Guardian owner state changed after prepare (prepared revision ${replacementRevision}, current revision ${ownerRevision}); abort the stale candidate before preparing again`)
|
|
452
493
|
if (request.ownerState === undefined || !isDeepStrictEqual(ownerAuthority(request.ownerState), replacementAuthority)) {
|
|
453
494
|
throw new Error("Prepared replacement state does not match the requested authority")
|
|
454
495
|
}
|
|
@@ -579,7 +620,7 @@ async function execute(request, socket) {
|
|
|
579
620
|
requireOwner(socket, request.command)
|
|
580
621
|
shuttingDown = true
|
|
581
622
|
try {
|
|
582
|
-
const results = await Promise.allSettled([...processes.values()].map((entry) => entry.process.stop()))
|
|
623
|
+
const results = await Promise.allSettled([...processes.values()].map((entry) => entry.process.stop(request.options)))
|
|
583
624
|
const errors = results.filter((result) => result.status === "rejected").map((result) => result.reason)
|
|
584
625
|
|
|
585
626
|
if (errors.length > 0) throw new AggregateError(errors, `Guardian failed to stop ${errors.length} owned process${errors.length === 1 ? "" : "es"}: ${errors.map((error) => errorMessage(error instanceof Error ? error : String(error))).join("; ")}`)
|
|
@@ -616,8 +657,17 @@ async function execute(request, socket) {
|
|
|
616
657
|
const existing = processes.get(request.key)
|
|
617
658
|
|
|
618
659
|
if (existing) {
|
|
619
|
-
if (existing.provenance
|
|
620
|
-
|
|
660
|
+
if (existing.provenance === request.provenance) return existing.process.status()
|
|
661
|
+
// A provenance mismatch only fences a process that still exists. When the registered
|
|
662
|
+
// child is conclusively gone (stopped or failed with no live pid), the old record is
|
|
663
|
+
// residue of a dead generation, and demanding its provenance wedges every later deploy
|
|
664
|
+
// against a process that no longer exists (TensorBuzz production, STC-695). The owner
|
|
665
|
+
// may re-register the key under the new definition; live processes stay fail-closed.
|
|
666
|
+
// A group that outlived its tracked process is not gone: it still holds the key's work,
|
|
667
|
+
// so re-registering it would start a second copy of a generation that is still running.
|
|
668
|
+
if (!childRecordIsGone(existing.process.status())) throw new Error(`Guardian provenance mismatch for ${request.key}`)
|
|
669
|
+
requireOwner(socket, request.command)
|
|
670
|
+
processes.delete(request.key)
|
|
621
671
|
}
|
|
622
672
|
|
|
623
673
|
const recoversLegacyProcess = Boolean(legacyGuardian && legacyKeys.has(request.key))
|
|
@@ -660,18 +710,6 @@ async function execute(request, socket) {
|
|
|
660
710
|
if (request.command === "start") {
|
|
661
711
|
record.desired = true
|
|
662
712
|
await record.process.start(request.reason, request.lifecycleRole)
|
|
663
|
-
} else if (request.command === "adopt") {
|
|
664
|
-
requireOwner(socket, request.command)
|
|
665
|
-
const requestedPid = request.pid
|
|
666
|
-
const requestedStartTime = request.processStartTime
|
|
667
|
-
const committedConfigDigest = ownerState && typeof ownerState === "object" && !Array.isArray(ownerState)
|
|
668
|
-
? /** @type {{authority?: {configDigest?: string}}} */ (ownerState).authority?.configDigest
|
|
669
|
-
: undefined
|
|
670
|
-
if (!request.configDigest || !committedConfigDigest || !isDeepStrictEqual(request.configDigest, committedConfigDigest)) throw new Error("Guardian process adoption authority does not match the committed owner")
|
|
671
|
-
if (!Number.isInteger(requestedPid) || !requestedStartTime) throw new Error("Guardian process adoption requires an exact PID and process start time")
|
|
672
|
-
if (typeof record.process.adopt !== "function") throw new Error("Guardian process adoption is not supported by this process implementation")
|
|
673
|
-
record.desired = true
|
|
674
|
-
await record.process.adopt({pid: /** @type {number} */ (requestedPid), processStartTime: requestedStartTime}, request.configDigest)
|
|
675
713
|
} else if (request.command === "activate") {
|
|
676
714
|
await record.process.activateStrict()
|
|
677
715
|
} else if (request.command === "reactivate" || request.command === "reactivate-with-command") {
|
|
@@ -720,6 +758,146 @@ async function execute(request, socket) {
|
|
|
720
758
|
return status
|
|
721
759
|
}
|
|
722
760
|
|
|
761
|
+
/**
|
|
762
|
+
* Names the mutation holding the owner-mutation fence: which mutation, which process,
|
|
763
|
+
* since when, and what it is blocked on. Refusals must be diagnosable without a live
|
|
764
|
+
* inspection of the daemon.
|
|
765
|
+
* @returns {string} Holder description used in refusal messages.
|
|
766
|
+
*/
|
|
767
|
+
function describeOwnerMutationLease() {
|
|
768
|
+
const startedAt = ownerMutationStartedAt ?? 0
|
|
769
|
+
const elapsedSeconds = Math.max(0, Math.round((Date.now() - startedAt) / 1000))
|
|
770
|
+
const holderPid = ownerMutationPid ?? ownerClientPid
|
|
771
|
+
|
|
772
|
+
return [
|
|
773
|
+
`owner mutation "${ownerMutationOperation || "unnamed operation"}"`,
|
|
774
|
+
`mutation ${ownerMutationId || "unknown"}`,
|
|
775
|
+
`held by process ${holderPid ?? "unknown"}`,
|
|
776
|
+
`since ${new Date(startedAt).toISOString()}`,
|
|
777
|
+
`${elapsedSeconds}s elapsed`,
|
|
778
|
+
`waiting for ${ownerMutationWaitingFor || "the operation to report completion"}`
|
|
779
|
+
].join("; ")
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
/**
|
|
783
|
+
* Reports the currently held owner-mutation lease.
|
|
784
|
+
* @returns {{elapsedMs: number, leaseMs: number, mutationId: string, operation: string, pid: number | null, startedAt: string, waitingFor: string} | null} Active lease, if any.
|
|
785
|
+
*/
|
|
786
|
+
function activeOwnerMutationStatus() {
|
|
787
|
+
if (!ownerMutationClient || !ownerMutationId) return null
|
|
788
|
+
|
|
789
|
+
const startedAt = ownerMutationStartedAt ?? Date.now()
|
|
790
|
+
|
|
791
|
+
return {
|
|
792
|
+
elapsedMs: Math.max(0, Date.now() - startedAt),
|
|
793
|
+
leaseMs: ownerMutationLeaseMs,
|
|
794
|
+
mutationId: ownerMutationId,
|
|
795
|
+
operation: ownerMutationOperation || "unnamed operation",
|
|
796
|
+
pid: ownerMutationPid ?? ownerClientPid ?? null,
|
|
797
|
+
startedAt: new Date(startedAt).toISOString(),
|
|
798
|
+
waitingFor: ownerMutationWaitingFor || "the operation to report completion"
|
|
799
|
+
}
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
/**
|
|
803
|
+
* Reports guardian identity and owner-mutation diagnostics.
|
|
804
|
+
* @returns {{guardianPid: number, ownerMutation: ReturnType<typeof activeOwnerMutationStatus>, version: string}} Guardian diagnostics.
|
|
805
|
+
*/
|
|
806
|
+
function guardianStatus() {
|
|
807
|
+
return {guardianPid: process.pid, ownerMutation: activeOwnerMutationStatus(), version: GUARDIAN_VERSION}
|
|
808
|
+
}
|
|
809
|
+
|
|
810
|
+
/** @returns {boolean} Whether the recorded owner-mutation holder is no longer alive. */
|
|
811
|
+
function reclaimDeadOwnerMutation() {
|
|
812
|
+
if (!ownerMutationClient || !ownerMutationPid || isProcessAlive(ownerMutationPid)) return false
|
|
813
|
+
|
|
814
|
+
const mutation = activeOwnerMutationStatus()
|
|
815
|
+
const deadOwnerSocket = ownerClient === ownerMutationClient && ownerClientPid === ownerMutationPid ? ownerClient : undefined
|
|
816
|
+
const diagnostic = {
|
|
817
|
+
code: "OWNER_MUTATION_RECLAIMED",
|
|
818
|
+
event: "owner-mutation-reclaimed",
|
|
819
|
+
mutationId: ownerMutationId,
|
|
820
|
+
operation: mutation?.operation || "unnamed operation",
|
|
821
|
+
pid: ownerMutationPid,
|
|
822
|
+
reason: "owner mutation holder process is no longer alive"
|
|
823
|
+
}
|
|
824
|
+
|
|
825
|
+
process.stderr.write(`${JSON.stringify({at: new Date().toISOString(), ...diagnostic})}\n`)
|
|
826
|
+
broadcast(diagnostic)
|
|
827
|
+
clearOwnerMutationLease()
|
|
828
|
+
if (deadOwnerSocket) releaseClosedOwner(deadOwnerSocket)
|
|
829
|
+
return true
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
/**
|
|
833
|
+
* Checks whether a process still exists without signaling it.
|
|
834
|
+
* @param {number} pid - Process id.
|
|
835
|
+
* @returns {boolean} Whether the process exists.
|
|
836
|
+
*/
|
|
837
|
+
function isProcessAlive(pid) {
|
|
838
|
+
try {
|
|
839
|
+
process.kill(pid, 0)
|
|
840
|
+
return true
|
|
841
|
+
} catch (error) {
|
|
842
|
+
return Boolean(error && typeof error === "object" && "code" in error && error.code === "EPERM")
|
|
843
|
+
}
|
|
844
|
+
}
|
|
845
|
+
|
|
846
|
+
/** @returns {void} */
|
|
847
|
+
function clearOwnerMutationLease() {
|
|
848
|
+
if (ownerMutationLeaseTimer) clearTimeout(ownerMutationLeaseTimer)
|
|
849
|
+
ownerMutationLeaseTimer = undefined
|
|
850
|
+
ownerMutationClient = undefined
|
|
851
|
+
ownerMutationId = undefined
|
|
852
|
+
ownerMutationOperation = undefined
|
|
853
|
+
ownerMutationWaitingFor = undefined
|
|
854
|
+
ownerMutationPid = undefined
|
|
855
|
+
ownerMutationStartedAt = undefined
|
|
856
|
+
ownerMutationLeaseMs = OWNER_MUTATION_LEASE_MS
|
|
857
|
+
}
|
|
858
|
+
|
|
859
|
+
/**
|
|
860
|
+
* Arms the lease bound for one mutation.
|
|
861
|
+
* @param {string} mutationId - Lease id the timer belongs to.
|
|
862
|
+
* @returns {void}
|
|
863
|
+
*/
|
|
864
|
+
function armOwnerMutationLease(mutationId) {
|
|
865
|
+
if (ownerMutationLeaseTimer) clearTimeout(ownerMutationLeaseTimer)
|
|
866
|
+
ownerMutationLeaseTimer = setTimeout(() => expireOwnerMutationLease(mutationId), ownerMutationLeaseMs)
|
|
867
|
+
ownerMutationLeaseTimer.unref()
|
|
868
|
+
}
|
|
869
|
+
|
|
870
|
+
/**
|
|
871
|
+
* A mutation that outruns its lease is reported and released instead of holding the fence
|
|
872
|
+
* forever, so a stuck mutation cannot lock out every later deploy. The holder learns the
|
|
873
|
+
* lease is gone when it tries to end it.
|
|
874
|
+
* @param {string} mutationId - Lease id the armed timer belongs to.
|
|
875
|
+
* @returns {void}
|
|
876
|
+
*/
|
|
877
|
+
function expireOwnerMutationLease(mutationId) {
|
|
878
|
+
ownerMutationLeaseTimer = undefined
|
|
879
|
+
if (!ownerMutationClient || ownerMutationId !== mutationId) return
|
|
880
|
+
const operation = ownerMutationOperation || "unnamed operation"
|
|
881
|
+
const leaseMs = ownerMutationLeaseMs
|
|
882
|
+
const elapsedMs = Math.max(0, Date.now() - (ownerMutationStartedAt ?? 0))
|
|
883
|
+
const diagnostic = {
|
|
884
|
+
code: "OWNER_MUTATION_LEASE_EXPIRED",
|
|
885
|
+
elapsedMs,
|
|
886
|
+
event: "owner-mutation-expired",
|
|
887
|
+
leaseMs,
|
|
888
|
+
mutationId,
|
|
889
|
+
operation,
|
|
890
|
+
pid: ownerMutationPid ?? ownerClientPid,
|
|
891
|
+
waitingFor: ownerMutationWaitingFor || "the operation to report completion"
|
|
892
|
+
}
|
|
893
|
+
|
|
894
|
+
process.stderr.write(`${JSON.stringify({at: new Date().toISOString(), ...diagnostic})}\n`)
|
|
895
|
+
if (!ownerMutationClient.destroyed) ownerMutationClient.write(`${JSON.stringify(diagnostic)}\n`)
|
|
896
|
+
broadcast(diagnostic)
|
|
897
|
+
ownerMutationExpired = {elapsedMs, leaseMs, mutationId, operation}
|
|
898
|
+
clearOwnerMutationLease()
|
|
899
|
+
}
|
|
900
|
+
|
|
723
901
|
/**
|
|
724
902
|
* @param {net.Socket} socket - Requesting client.
|
|
725
903
|
* @param {string} command - Command name.
|
|
@@ -740,10 +918,7 @@ function releaseClosedOwner(socket) {
|
|
|
740
918
|
rememberDisconnectedOwnerSuccessors(socket)
|
|
741
919
|
ownerClient = undefined
|
|
742
920
|
ownerClientPid = undefined
|
|
743
|
-
if (ownerMutationClient === socket)
|
|
744
|
-
ownerMutationClient = undefined
|
|
745
|
-
ownerMutationId = undefined
|
|
746
|
-
}
|
|
921
|
+
if (ownerMutationClient === socket) clearOwnerMutationLease()
|
|
747
922
|
if (replacementClient && replacementOwnerState) commitReplacement()
|
|
748
923
|
else {
|
|
749
924
|
if (incompleteRestartChild) {
|
package/src/process-memory.js
CHANGED
|
@@ -4,8 +4,14 @@ import fs from "node:fs"
|
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* @typedef {{command: string, pid: number, rssBytes: number | undefined}} ProcessGroupMember
|
|
7
|
+
* @typedef {{cpuMs: number | undefined, liveMemberCount: number, memberCount: number}} ProcessGroupSample
|
|
7
8
|
*/
|
|
8
9
|
|
|
10
|
+
// Linux reports /proc/<pid>/stat CPU time in clock ticks and keeps USER_HZ at 100 for
|
|
11
|
+
// userspace, so ticks * 10 / 1 is milliseconds. This is a monotonic per-process counter,
|
|
12
|
+
// not a rate: two reads of it measure whether a process did any work between them.
|
|
13
|
+
const CLOCK_TICKS_PER_MS = 10
|
|
14
|
+
|
|
9
15
|
/**
|
|
10
16
|
* Lists the members of a managed process group with each member's resident memory.
|
|
11
17
|
* Rollbridge spawns each process detached, so the spawned pid is the process-group
|
|
@@ -71,41 +77,61 @@ export function processGroupHasLiveMembers(pgid, procPath = "/proc") {
|
|
|
71
77
|
}
|
|
72
78
|
|
|
73
79
|
/**
|
|
74
|
-
*
|
|
75
|
-
* @param {number}
|
|
80
|
+
* Measures the total resident memory (RSS) of a managed process group.
|
|
81
|
+
* @param {number} pgid - Process-group id (the detached spawn's pid).
|
|
82
|
+
* @returns {number | undefined} Total resident memory in bytes, or undefined when unmeasurable.
|
|
83
|
+
*/
|
|
84
|
+
export function measureProcessGroupRssBytes(pgid) {
|
|
85
|
+
const measured = processGroupMembers(pgid).filter((member) => member.rssBytes !== undefined)
|
|
86
|
+
|
|
87
|
+
if (measured.length === 0) return undefined
|
|
88
|
+
|
|
89
|
+
return measured.reduce((total, member) => total + (member.rssBytes ?? 0), 0)
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Samples one managed process group for drain liveness: how much CPU its members have
|
|
94
|
+
* consumed in total, and how many of them can still run.
|
|
95
|
+
*
|
|
96
|
+
* CPU is the sum of every member's cumulative user+system time, defunct members included:
|
|
97
|
+
* a member that exits takes its consumed time out of the sum, so the difference between two
|
|
98
|
+
* samples is non-zero whenever the group changed at all — new work or a departing member.
|
|
99
|
+
* Only an exactly unchanged difference says the group did nothing between the two reads.
|
|
100
|
+
* @param {number} pgid - Process-group id (the detached spawn's pid).
|
|
76
101
|
* @param {string} [procPath] - Procfs root (overridable for deterministic tests).
|
|
77
|
-
* @returns {
|
|
102
|
+
* @returns {ProcessGroupSample | undefined} The sample, or undefined only when procfs cannot be
|
|
103
|
+
* read at all. A group with no members left is a sample of zero, not a missing measurement.
|
|
78
104
|
*/
|
|
79
|
-
export function
|
|
80
|
-
|
|
105
|
+
export function sampleProcessGroup(pgid, procPath = "/proc") {
|
|
106
|
+
/** @type {string[]} */
|
|
107
|
+
let entries
|
|
81
108
|
|
|
82
109
|
try {
|
|
83
|
-
|
|
110
|
+
entries = fs.readdirSync(procPath)
|
|
84
111
|
} catch {
|
|
85
112
|
return undefined
|
|
86
113
|
}
|
|
87
114
|
|
|
88
|
-
|
|
115
|
+
let cpuMs = 0
|
|
116
|
+
let liveMemberCount = 0
|
|
117
|
+
let memberCount = 0
|
|
118
|
+
let measurable = true
|
|
89
119
|
|
|
90
|
-
|
|
120
|
+
for (const entry of entries) {
|
|
121
|
+
if (!/^\d+$/.test(entry)) continue
|
|
91
122
|
|
|
92
|
-
|
|
93
|
-
const startTime = fields[19]
|
|
123
|
+
const stat = processStat(entry, procPath)
|
|
94
124
|
|
|
95
|
-
|
|
96
|
-
}
|
|
125
|
+
if (stat?.pgrp !== pgid) continue
|
|
97
126
|
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
* @param {number} pgid - Process-group id (the detached spawn's pid).
|
|
101
|
-
* @returns {number | undefined} Total resident memory in bytes, or undefined when unmeasurable.
|
|
102
|
-
*/
|
|
103
|
-
export function measureProcessGroupRssBytes(pgid) {
|
|
104
|
-
const measured = processGroupMembers(pgid).filter((member) => member.rssBytes !== undefined)
|
|
127
|
+
memberCount += 1
|
|
128
|
+
if (stat.state !== "Z" && stat.state !== "X") liveMemberCount += 1
|
|
105
129
|
|
|
106
|
-
|
|
130
|
+
if (stat.cpuMs === undefined) measurable = false
|
|
131
|
+
else cpuMs += stat.cpuMs
|
|
132
|
+
}
|
|
107
133
|
|
|
108
|
-
return
|
|
134
|
+
return {cpuMs: measurable ? cpuMs : undefined, liveMemberCount, memberCount}
|
|
109
135
|
}
|
|
110
136
|
|
|
111
137
|
/**
|
|
@@ -131,7 +157,7 @@ function processGroupId(pid) {
|
|
|
131
157
|
/**
|
|
132
158
|
* @param {string} pid - Process id.
|
|
133
159
|
* @param {string} procPath - Procfs root.
|
|
134
|
-
* @returns {{pgrp: number, state: string} | undefined} Parsed process state and
|
|
160
|
+
* @returns {{cpuMs: number | undefined, pgrp: number, state: string} | undefined} Parsed process state, group, and cumulative CPU.
|
|
135
161
|
*/
|
|
136
162
|
function processStat(pid, procPath) {
|
|
137
163
|
let stat
|
|
@@ -151,7 +177,16 @@ function processStat(pid, procPath) {
|
|
|
151
177
|
const fields = stat.slice(commEnd + 2).split(" ")
|
|
152
178
|
const pgrp = Number(fields[2])
|
|
153
179
|
|
|
154
|
-
|
|
180
|
+
if (!Number.isInteger(pgrp)) return undefined
|
|
181
|
+
|
|
182
|
+
// utime and stime are fields 14 and 15 of stat, i.e. index 11 and 12 after the comm field.
|
|
183
|
+
const utimeTicks = Number(fields[11])
|
|
184
|
+
const stimeTicks = Number(fields[12])
|
|
185
|
+
const cpuMs = Number.isFinite(utimeTicks) && Number.isFinite(stimeTicks)
|
|
186
|
+
? (utimeTicks + stimeTicks) * CLOCK_TICKS_PER_MS
|
|
187
|
+
: undefined
|
|
188
|
+
|
|
189
|
+
return {cpuMs, pgrp, state: fields[0]}
|
|
155
190
|
}
|
|
156
191
|
|
|
157
192
|
/**
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* @typedef {import("./config.js").ProcessConfig} ProcessConfig
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
const POLICY_START_RANK = /** @type {Record<ProcessConfig["policy"], number>} */ ({companion: 1, proxied: 2, service: 0, singleton: 3})
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Order in which Rollbridge starts processes, which is also the only order a `dependsOn`
|
|
11
|
+
* edge may point backwards in: daemon-wide persistent services are started before every
|
|
12
|
+
* release generation, then handoff services, companions, and the proxied process last.
|
|
13
|
+
* Config order is kept inside each group.
|
|
14
|
+
* @param {ProcessConfig[]} processes - Normalized processes.
|
|
15
|
+
* @param {{includePersistentServices?: boolean}} [options] - Whether to include the
|
|
16
|
+
* daemon-wide persistent services that a release group never starts itself.
|
|
17
|
+
* @returns {ProcessConfig[]} Ordered process configs.
|
|
18
|
+
*/
|
|
19
|
+
export function processStartOrder(processes, {includePersistentServices = false} = {}) {
|
|
20
|
+
const persistentServices = includePersistentServices
|
|
21
|
+
? processes.filter((processConfig) => processConfig.policy === "service" && processConfig.deployStrategy === "persistent")
|
|
22
|
+
: []
|
|
23
|
+
const generationProcesses = processes.filter((processConfig) => processConfig.policy !== "singleton" && (processConfig.policy !== "service" || processConfig.deployStrategy === "handoff"))
|
|
24
|
+
|
|
25
|
+
return [...persistentServices, ...generationProcesses.sort((left, right) => POLICY_START_RANK[left.policy] - POLICY_START_RANK[right.policy])]
|
|
26
|
+
}
|