rollbridge 0.1.73 → 0.1.75
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +63 -10
- package/TODO.md +13 -0
- package/changelog.d/20261025-guardian-re-register-dead-generation.md +13 -0
- package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +1 -0
- package/changelog.d/20261104-compose-service-units.md +21 -0
- package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +20 -0
- package/changelog.d/20261106-release-dependency-boot-budgets.md +32 -0
- package/changelog.d/20261109-path-filtered-builds.md +11 -0
- package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +26 -0
- package/changelog.d/20261111-contention-sensitive-daemon-specs.md +27 -0
- package/changelog.d/20261113-bounded-nonblocking-retirement.md +5 -0
- package/changelog.d/20261115-bound-draining-release-groups.md +62 -0
- package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +27 -0
- package/changelog.d/20261115-wedged-generation-escalation.md +44 -0
- package/changelog.d/20261116-bounded-explicit-stop-retirement.md +30 -0
- package/changelog.d/20261116-config-json-canonical-shape.md +11 -0
- package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +29 -0
- package/changelog.d/20261116-guardian-version-handoff.md +37 -0
- package/changelog.d/20261116-reclaim-dead-owner-mutation.md +7 -0
- package/changelog.d/20261117-activate-before-incumbent-retirement.md +1 -0
- package/changelog.d/20261117-activation-path-drain-deadline.md +35 -0
- package/changelog.d/20261117-stop-companion-after-retired-dependency.md +7 -0
- package/changelog.d/20261118-bounded-control-client-close.md +4 -0
- package/changelog.d/20261118-incumbent-retirement-authority-guard.md +1 -0
- package/changelog.d/20261118-owner-retirement-no-guardian-bound.md +19 -0
- package/changelog.d/20261118-shutdown-retirement-bound-restored.md +31 -0
- package/docs/cli.md +128 -3
- package/docs/compose-services.md +222 -0
- package/docs/config.md +177 -6
- package/docs/troubleshooting.md +97 -6
- package/docs/velocious.md +1 -1
- package/docs/workers.md +11 -0
- package/examples/tensorbuzz.com.js +3 -2
- package/package.json +1 -1
- package/src/cli.js +69 -3
- package/src/compose-unit.js +725 -0
- package/src/config.js +346 -8
- package/src/control-client.js +32 -5
- package/src/daemon.js +448 -76
- package/src/doctor.js +74 -0
- package/src/drain-policy.js +252 -0
- package/src/guardian-client.js +92 -9
- package/src/health.js +99 -8
- package/src/managed-process.js +197 -16
- package/src/process-guardian.js +212 -22
- package/src/process-memory.js +62 -2
- package/src/process-order.js +26 -0
- package/src/release-group.js +452 -39
- package/tensorbuzz.yml +20 -1
- package/test/completion.test.js +5 -2
- package/test/compose-config.test.js +160 -0
- package/test/compose-docker.test.js +297 -0
- package/test/compose-unit.test.js +415 -0
- package/test/config-validation.test.js +135 -5
- package/test/control-protocol.test.js +22 -0
- package/test/drain-policy.test.js +328 -0
- package/test/fixtures/deferred-boot-app.js +106 -0
- package/test/fixtures/gated-app.js +50 -0
- package/test/guardian-client.test.js +149 -5
- package/test/health.test.js +74 -1
- package/test/managed-process.test.js +260 -1
- package/test/owner-recovery.test.js +151 -20
- package/test/owner-replacement.test.js +176 -15
- package/test/process-memory.test.js +47 -1
- package/test/release-group.test.js +447 -2
- package/test/rollbridge.test.js +774 -72
- package/test/shutdown-completion.test.js +119 -0
- package/test/shutdown-retirement-bound.test.js +162 -0
- package/test/support/process.js +3 -1
- package/test/tensorbuzz-config.test.js +228 -0
package/src/process-guardian.js
CHANGED
|
@@ -4,14 +4,21 @@ import fsSync from "node:fs"
|
|
|
4
4
|
import fs from "node:fs/promises"
|
|
5
5
|
import net from "node:net"
|
|
6
6
|
import crypto from "node:crypto"
|
|
7
|
+
import {createRequire} from "node:module"
|
|
7
8
|
import {spawn} from "node:child_process"
|
|
8
9
|
import {isDeepStrictEqual} from "node:util"
|
|
9
10
|
import path from "node:path"
|
|
10
11
|
import GuardianClient from "./guardian-client.js"
|
|
11
|
-
import ManagedProcess from "./managed-process.js"
|
|
12
|
+
import ManagedProcess, {childRecordIsGone} from "./managed-process.js"
|
|
12
13
|
import {writeState} from "./state-store.js"
|
|
13
14
|
|
|
14
15
|
const OWNER_RESTART_RETRY_MS = 1000
|
|
16
|
+
const commonJsRequire = createRequire(import.meta.url)
|
|
17
|
+
const GUARDIAN_VERSION = /** @type {{version: string}} */ (commonJsRequire("../package.json")).version
|
|
18
|
+
// An owner mutation is deploy/stop/restart/rollback/recovery work. A retained
|
|
19
|
+
// generation may legitimately drain for hours, but a mutation that holds the
|
|
20
|
+
// fence that long is stuck, so the lease is bounded and released on expiry.
|
|
21
|
+
const OWNER_MUTATION_LEASE_MS = 900000
|
|
15
22
|
|
|
16
23
|
/** @typedef {import("node:child_process").ChildProcess["signalCode"]} ChildExitSignal */
|
|
17
24
|
|
|
@@ -24,7 +31,7 @@ const OWNER_RESTART_RETRY_MS = 1000
|
|
|
24
31
|
* @property {number} id - Request id.
|
|
25
32
|
* @property {string} [key] - Stable process key.
|
|
26
33
|
* @property {boolean} [localSource] - Whether the retired daemon physically owns the published source.
|
|
27
|
-
* @property {{timeoutMs?: number}} [options] - Stop options.
|
|
34
|
+
* @property {{retirementTimeoutMs?: number, timeoutMs?: number}} [options] - Stop options.
|
|
28
35
|
* @property {string} [previousProvenance] - Expected current provenance.
|
|
29
36
|
* @property {string} [provenance] - New or registered provenance.
|
|
30
37
|
* @property {string} [releaseId] - Retained release identity.
|
|
@@ -32,7 +39,9 @@ const OWNER_RESTART_RETRY_MS = 1000
|
|
|
32
39
|
* @property {import("./json.js").JsonValue} [ownerState] - Private owner transfer state.
|
|
33
40
|
* @property {string} [replacementId] - Prepared replacement transaction id.
|
|
34
41
|
* @property {string} [mutationId] - Owner mutation lease id.
|
|
42
|
+
* @property {number} [mutationTimeoutMs] - Bound for the owner mutation lease.
|
|
35
43
|
* @property {string} [operation] - Owner mutation diagnostic name.
|
|
44
|
+
* @property {string} [waitingFor] - What the held owner mutation is blocked on.
|
|
36
45
|
* @property {number} [ownerPid] - Exact claimant process PID.
|
|
37
46
|
* @property {string} [configDigest] - Exact committed owner configuration digest.
|
|
38
47
|
* @property {number} [pid] - Exact live process PID to re-adopt.
|
|
@@ -106,6 +115,19 @@ const legacyKeys = legacyGuardian ? committedOwnerProcessKeys(ownerState) : new
|
|
|
106
115
|
let ownerMutationClient
|
|
107
116
|
/** @type {string | undefined} */
|
|
108
117
|
let ownerMutationId
|
|
118
|
+
/** @type {string | undefined} */
|
|
119
|
+
let ownerMutationOperation
|
|
120
|
+
/** @type {string | undefined} */
|
|
121
|
+
let ownerMutationWaitingFor
|
|
122
|
+
/** @type {number | undefined} */
|
|
123
|
+
let ownerMutationPid
|
|
124
|
+
/** @type {number | undefined} */
|
|
125
|
+
let ownerMutationStartedAt
|
|
126
|
+
let ownerMutationLeaseMs = OWNER_MUTATION_LEASE_MS
|
|
127
|
+
/** @type {ReturnType<typeof setTimeout> | undefined} */
|
|
128
|
+
let ownerMutationLeaseTimer
|
|
129
|
+
/** @type {{elapsedMs: number, leaseMs: number, mutationId: string, operation: string} | undefined} */
|
|
130
|
+
let ownerMutationExpired
|
|
109
131
|
/** @type {net.Socket | undefined} */
|
|
110
132
|
let retiringClient
|
|
111
133
|
/** @type {string | undefined} */
|
|
@@ -157,10 +179,7 @@ const server = net.createServer((socket) => {
|
|
|
157
179
|
clearTimeout(waiter.timer)
|
|
158
180
|
waiter.reject(new Error("Owner claimant disconnected"))
|
|
159
181
|
}
|
|
160
|
-
if (ownerMutationClient === socket
|
|
161
|
-
ownerMutationClient = undefined
|
|
162
|
-
ownerMutationId = undefined
|
|
163
|
-
}
|
|
182
|
+
if (ownerMutationClient === socket) clearOwnerMutationLease()
|
|
164
183
|
if (controlLessRetirement) {
|
|
165
184
|
const preCommitHandoff = replacementClient && ownerClient === socket && retiringReplacementId === replacementId
|
|
166
185
|
|
|
@@ -262,13 +281,14 @@ async function handleLine(socket, line) {
|
|
|
262
281
|
async function execute(request, socket) {
|
|
263
282
|
if (shuttingDown) throw new Error("Process guardian is shutting down")
|
|
264
283
|
|
|
265
|
-
if (request.command === "capabilities") return {daemonRecovery: 1, generationReactivation: 1}
|
|
284
|
+
if (request.command === "capabilities") return {daemonRecovery: 1, generationReactivation: 1, guardianVersion: GUARDIAN_VERSION}
|
|
266
285
|
|
|
267
286
|
if (request.command === "owner-replacement-capabilities") {
|
|
268
287
|
return {commands: ["commit-retired-owner-replacement"], protocol: "owner-replacement", version: 1}
|
|
269
288
|
}
|
|
270
289
|
|
|
271
290
|
if (request.command === "claim-owner") {
|
|
291
|
+
reclaimDeadOwnerMutation()
|
|
272
292
|
if (retirementFailed) throw new Error("Guardian owner replacement is fenced after incomplete retired-listener state transfer")
|
|
273
293
|
if (ownerState !== undefined && !isDeepStrictEqual(request.authority, ownerAuthority(ownerState))) {
|
|
274
294
|
const waitsForPreparedAuthority = Boolean(ownerClient && replacementOwnerState && isDeepStrictEqual(request.authority, replacementAuthority))
|
|
@@ -304,8 +324,7 @@ async function execute(request, socket) {
|
|
|
304
324
|
rememberDisconnectedOwnerSuccessors(socket)
|
|
305
325
|
ownerClient = undefined
|
|
306
326
|
ownerClientPid = undefined
|
|
307
|
-
|
|
308
|
-
ownerMutationId = undefined
|
|
327
|
+
clearOwnerMutationLease()
|
|
309
328
|
ownerRevision += 1
|
|
310
329
|
grantNextOwner()
|
|
311
330
|
return {retired: true}
|
|
@@ -397,29 +416,54 @@ async function execute(request, socket) {
|
|
|
397
416
|
return {committedReplacementId: committedReplacementId ?? null, ownerClaimed: Boolean(ownerClient), retirementFailed, retirementPending: Boolean(retiringClient), retirementReady: retiringListenerReady}
|
|
398
417
|
}
|
|
399
418
|
|
|
419
|
+
if (request.command === "guardian-status") return guardianStatus()
|
|
420
|
+
|
|
400
421
|
if (request.command === "begin-owner-mutation") {
|
|
401
422
|
requireOwner(socket, request.command)
|
|
423
|
+
reclaimDeadOwnerMutation()
|
|
402
424
|
if (replacementClient) throw new Error(`Owner mutation ${request.operation || "operation"} is fenced while an owner replacement is prepared`)
|
|
403
|
-
|
|
425
|
+
const leaseMs = request.mutationTimeoutMs === undefined ? OWNER_MUTATION_LEASE_MS : request.mutationTimeoutMs
|
|
426
|
+
|
|
427
|
+
if (!Number.isInteger(leaseMs) || leaseMs <= 0) throw new Error("Owner mutation lease bound must be a positive number of milliseconds")
|
|
428
|
+
if (ownerMutationClient) throw new Error(`Another owner mutation is already in progress: ${describeOwnerMutationLease()}; refused ${request.operation || "operation"}`)
|
|
404
429
|
ownerMutationClient = socket
|
|
405
430
|
ownerMutationId = crypto.randomUUID()
|
|
406
|
-
|
|
431
|
+
ownerMutationOperation = request.operation
|
|
432
|
+
ownerMutationWaitingFor = request.waitingFor
|
|
433
|
+
ownerMutationPid = request.ownerPid ?? ownerClientPid
|
|
434
|
+
ownerMutationStartedAt = Date.now()
|
|
435
|
+
ownerMutationLeaseMs = leaseMs
|
|
436
|
+
armOwnerMutationLease(ownerMutationId)
|
|
437
|
+
return {leaseMs, mutationId: ownerMutationId}
|
|
407
438
|
}
|
|
408
439
|
|
|
409
440
|
if (request.command === "end-owner-mutation") {
|
|
410
441
|
requireOwner(socket, request.command)
|
|
442
|
+
if (ownerMutationExpired && request.mutationId === ownerMutationExpired.mutationId) {
|
|
443
|
+
const expired = ownerMutationExpired
|
|
444
|
+
|
|
445
|
+
ownerMutationExpired = undefined
|
|
446
|
+
throw new Error(`Owner mutation ${expired.operation} exceeded its ${expired.leaseMs} ms lease and was released after ${expired.elapsedMs} ms; the mutation failed and its lease is gone`)
|
|
447
|
+
}
|
|
411
448
|
if (ownerMutationClient !== socket || request.mutationId !== ownerMutationId) throw new Error("Owner mutation lease does not match the active mutation")
|
|
412
|
-
|
|
413
|
-
ownerMutationId = undefined
|
|
449
|
+
clearOwnerMutationLease()
|
|
414
450
|
return {ended: true}
|
|
415
451
|
}
|
|
416
452
|
|
|
453
|
+
if (request.command === "report-owner-mutation-wait") {
|
|
454
|
+
requireOwner(socket, request.command)
|
|
455
|
+
if (ownerMutationClient !== socket || request.mutationId !== ownerMutationId) throw new Error("Owner mutation lease does not match the active mutation")
|
|
456
|
+
if (typeof request.waitingFor !== "string" || !request.waitingFor.trim()) throw new Error("Owner mutation wait report requires a waitingFor description")
|
|
457
|
+
ownerMutationWaitingFor = request.waitingFor.trim()
|
|
458
|
+
return {reported: true}
|
|
459
|
+
}
|
|
460
|
+
|
|
417
461
|
if (request.command === "prepare-owner-replacement") {
|
|
418
462
|
if (retirementFailed) throw new Error("Guardian owner replacement is fenced after incomplete retired-listener state transfer")
|
|
419
463
|
if (retiringClient) throw new Error("Guardian owner replacement cannot prepare while listener retirement is pending")
|
|
420
464
|
if (socket === ownerClient) throw new Error("Committed owner cannot prepare itself as a replacement")
|
|
421
465
|
if (replacementClient && replacementClient !== socket) throw new Error("Another owner replacement candidate is already prepared")
|
|
422
|
-
if (ownerMutationClient) throw new Error(
|
|
466
|
+
if (ownerMutationClient) throw new Error(`Owner replacement cannot prepare while an owner mutation is in progress: ${describeOwnerMutationLease()}`)
|
|
423
467
|
if (!ownerState) throw new Error("Committed owner has not published transferable state")
|
|
424
468
|
const currentAuthority = ownerAuthority(ownerState)
|
|
425
469
|
const resumesCommittedAuthority = !ownerClient && isDeepStrictEqual(currentAuthority, request.nextAuthority)
|
|
@@ -448,7 +492,7 @@ async function execute(request, socket) {
|
|
|
448
492
|
|
|
449
493
|
if (request.command === "stage-owner-replacement") {
|
|
450
494
|
requireReplacement(socket, request)
|
|
451
|
-
if (replacementRevision !== ownerRevision) throw new Error(
|
|
495
|
+
if (replacementRevision !== ownerRevision) throw new Error(`Guardian owner state changed after prepare (prepared revision ${replacementRevision}, current revision ${ownerRevision}); abort the stale candidate before preparing again`)
|
|
452
496
|
if (request.ownerState === undefined || !isDeepStrictEqual(ownerAuthority(request.ownerState), replacementAuthority)) {
|
|
453
497
|
throw new Error("Prepared replacement state does not match the requested authority")
|
|
454
498
|
}
|
|
@@ -579,7 +623,7 @@ async function execute(request, socket) {
|
|
|
579
623
|
requireOwner(socket, request.command)
|
|
580
624
|
shuttingDown = true
|
|
581
625
|
try {
|
|
582
|
-
const results = await Promise.allSettled([...processes.values()].map((entry) => entry.process.stop()))
|
|
626
|
+
const results = await Promise.allSettled([...processes.values()].map((entry) => entry.process.stop(request.options)))
|
|
583
627
|
const errors = results.filter((result) => result.status === "rejected").map((result) => result.reason)
|
|
584
628
|
|
|
585
629
|
if (errors.length > 0) throw new AggregateError(errors, `Guardian failed to stop ${errors.length} owned process${errors.length === 1 ? "" : "es"}: ${errors.map((error) => errorMessage(error instanceof Error ? error : String(error))).join("; ")}`)
|
|
@@ -616,8 +660,17 @@ async function execute(request, socket) {
|
|
|
616
660
|
const existing = processes.get(request.key)
|
|
617
661
|
|
|
618
662
|
if (existing) {
|
|
619
|
-
if (existing.provenance
|
|
620
|
-
|
|
663
|
+
if (existing.provenance === request.provenance) return existing.process.status()
|
|
664
|
+
// A provenance mismatch only fences a process that still exists. When the registered
|
|
665
|
+
// child is conclusively gone (stopped or failed with no live pid), the old record is
|
|
666
|
+
// residue of a dead generation, and demanding its provenance wedges every later deploy
|
|
667
|
+
// against a process that no longer exists (TensorBuzz production, STC-695). The owner
|
|
668
|
+
// may re-register the key under the new definition; live processes stay fail-closed.
|
|
669
|
+
// A group that outlived its tracked process is not gone: it still holds the key's work,
|
|
670
|
+
// so re-registering it would start a second copy of a generation that is still running.
|
|
671
|
+
if (!childRecordIsGone(existing.process.status())) throw new Error(`Guardian provenance mismatch for ${request.key}`)
|
|
672
|
+
requireOwner(socket, request.command)
|
|
673
|
+
processes.delete(request.key)
|
|
621
674
|
}
|
|
622
675
|
|
|
623
676
|
const recoversLegacyProcess = Boolean(legacyGuardian && legacyKeys.has(request.key))
|
|
@@ -720,6 +773,146 @@ async function execute(request, socket) {
|
|
|
720
773
|
return status
|
|
721
774
|
}
|
|
722
775
|
|
|
776
|
+
/**
|
|
777
|
+
* Names the mutation holding the owner-mutation fence: which mutation, which process,
|
|
778
|
+
* since when, and what it is blocked on. Refusals must be diagnosable without a live
|
|
779
|
+
* inspection of the daemon.
|
|
780
|
+
* @returns {string} Holder description used in refusal messages.
|
|
781
|
+
*/
|
|
782
|
+
function describeOwnerMutationLease() {
|
|
783
|
+
const startedAt = ownerMutationStartedAt ?? 0
|
|
784
|
+
const elapsedSeconds = Math.max(0, Math.round((Date.now() - startedAt) / 1000))
|
|
785
|
+
const holderPid = ownerMutationPid ?? ownerClientPid
|
|
786
|
+
|
|
787
|
+
return [
|
|
788
|
+
`owner mutation "${ownerMutationOperation || "unnamed operation"}"`,
|
|
789
|
+
`mutation ${ownerMutationId || "unknown"}`,
|
|
790
|
+
`held by process ${holderPid ?? "unknown"}`,
|
|
791
|
+
`since ${new Date(startedAt).toISOString()}`,
|
|
792
|
+
`${elapsedSeconds}s elapsed`,
|
|
793
|
+
`waiting for ${ownerMutationWaitingFor || "the operation to report completion"}`
|
|
794
|
+
].join("; ")
|
|
795
|
+
}
|
|
796
|
+
|
|
797
|
+
/**
|
|
798
|
+
* Reports the currently held owner-mutation lease.
|
|
799
|
+
* @returns {{elapsedMs: number, leaseMs: number, mutationId: string, operation: string, pid: number | null, startedAt: string, waitingFor: string} | null} Active lease, if any.
|
|
800
|
+
*/
|
|
801
|
+
function activeOwnerMutationStatus() {
|
|
802
|
+
if (!ownerMutationClient || !ownerMutationId) return null
|
|
803
|
+
|
|
804
|
+
const startedAt = ownerMutationStartedAt ?? Date.now()
|
|
805
|
+
|
|
806
|
+
return {
|
|
807
|
+
elapsedMs: Math.max(0, Date.now() - startedAt),
|
|
808
|
+
leaseMs: ownerMutationLeaseMs,
|
|
809
|
+
mutationId: ownerMutationId,
|
|
810
|
+
operation: ownerMutationOperation || "unnamed operation",
|
|
811
|
+
pid: ownerMutationPid ?? ownerClientPid ?? null,
|
|
812
|
+
startedAt: new Date(startedAt).toISOString(),
|
|
813
|
+
waitingFor: ownerMutationWaitingFor || "the operation to report completion"
|
|
814
|
+
}
|
|
815
|
+
}
|
|
816
|
+
|
|
817
|
+
/**
|
|
818
|
+
* Reports guardian identity and owner-mutation diagnostics.
|
|
819
|
+
* @returns {{guardianPid: number, ownerMutation: ReturnType<typeof activeOwnerMutationStatus>, version: string}} Guardian diagnostics.
|
|
820
|
+
*/
|
|
821
|
+
function guardianStatus() {
|
|
822
|
+
return {guardianPid: process.pid, ownerMutation: activeOwnerMutationStatus(), version: GUARDIAN_VERSION}
|
|
823
|
+
}
|
|
824
|
+
|
|
825
|
+
/** @returns {boolean} Whether the recorded owner-mutation holder is no longer alive. */
|
|
826
|
+
function reclaimDeadOwnerMutation() {
|
|
827
|
+
if (!ownerMutationClient || !ownerMutationPid || isProcessAlive(ownerMutationPid)) return false
|
|
828
|
+
|
|
829
|
+
const mutation = activeOwnerMutationStatus()
|
|
830
|
+
const deadOwnerSocket = ownerClient === ownerMutationClient && ownerClientPid === ownerMutationPid ? ownerClient : undefined
|
|
831
|
+
const diagnostic = {
|
|
832
|
+
code: "OWNER_MUTATION_RECLAIMED",
|
|
833
|
+
event: "owner-mutation-reclaimed",
|
|
834
|
+
mutationId: ownerMutationId,
|
|
835
|
+
operation: mutation?.operation || "unnamed operation",
|
|
836
|
+
pid: ownerMutationPid,
|
|
837
|
+
reason: "owner mutation holder process is no longer alive"
|
|
838
|
+
}
|
|
839
|
+
|
|
840
|
+
process.stderr.write(`${JSON.stringify({at: new Date().toISOString(), ...diagnostic})}\n`)
|
|
841
|
+
broadcast(diagnostic)
|
|
842
|
+
clearOwnerMutationLease()
|
|
843
|
+
if (deadOwnerSocket) releaseClosedOwner(deadOwnerSocket)
|
|
844
|
+
return true
|
|
845
|
+
}
|
|
846
|
+
|
|
847
|
+
/**
|
|
848
|
+
* Checks whether a process still exists without signaling it.
|
|
849
|
+
* @param {number} pid - Process id.
|
|
850
|
+
* @returns {boolean} Whether the process exists.
|
|
851
|
+
*/
|
|
852
|
+
function isProcessAlive(pid) {
|
|
853
|
+
try {
|
|
854
|
+
process.kill(pid, 0)
|
|
855
|
+
return true
|
|
856
|
+
} catch (error) {
|
|
857
|
+
return Boolean(error && typeof error === "object" && "code" in error && error.code === "EPERM")
|
|
858
|
+
}
|
|
859
|
+
}
|
|
860
|
+
|
|
861
|
+
/** @returns {void} */
|
|
862
|
+
function clearOwnerMutationLease() {
|
|
863
|
+
if (ownerMutationLeaseTimer) clearTimeout(ownerMutationLeaseTimer)
|
|
864
|
+
ownerMutationLeaseTimer = undefined
|
|
865
|
+
ownerMutationClient = undefined
|
|
866
|
+
ownerMutationId = undefined
|
|
867
|
+
ownerMutationOperation = undefined
|
|
868
|
+
ownerMutationWaitingFor = undefined
|
|
869
|
+
ownerMutationPid = undefined
|
|
870
|
+
ownerMutationStartedAt = undefined
|
|
871
|
+
ownerMutationLeaseMs = OWNER_MUTATION_LEASE_MS
|
|
872
|
+
}
|
|
873
|
+
|
|
874
|
+
/**
|
|
875
|
+
* Arms the lease bound for one mutation.
|
|
876
|
+
* @param {string} mutationId - Lease id the timer belongs to.
|
|
877
|
+
* @returns {void}
|
|
878
|
+
*/
|
|
879
|
+
function armOwnerMutationLease(mutationId) {
|
|
880
|
+
if (ownerMutationLeaseTimer) clearTimeout(ownerMutationLeaseTimer)
|
|
881
|
+
ownerMutationLeaseTimer = setTimeout(() => expireOwnerMutationLease(mutationId), ownerMutationLeaseMs)
|
|
882
|
+
ownerMutationLeaseTimer.unref()
|
|
883
|
+
}
|
|
884
|
+
|
|
885
|
+
/**
|
|
886
|
+
* A mutation that outruns its lease is reported and released instead of holding the fence
|
|
887
|
+
* forever, so a stuck mutation cannot lock out every later deploy. The holder learns the
|
|
888
|
+
* lease is gone when it tries to end it.
|
|
889
|
+
* @param {string} mutationId - Lease id the armed timer belongs to.
|
|
890
|
+
* @returns {void}
|
|
891
|
+
*/
|
|
892
|
+
function expireOwnerMutationLease(mutationId) {
|
|
893
|
+
ownerMutationLeaseTimer = undefined
|
|
894
|
+
if (!ownerMutationClient || ownerMutationId !== mutationId) return
|
|
895
|
+
const operation = ownerMutationOperation || "unnamed operation"
|
|
896
|
+
const leaseMs = ownerMutationLeaseMs
|
|
897
|
+
const elapsedMs = Math.max(0, Date.now() - (ownerMutationStartedAt ?? 0))
|
|
898
|
+
const diagnostic = {
|
|
899
|
+
code: "OWNER_MUTATION_LEASE_EXPIRED",
|
|
900
|
+
elapsedMs,
|
|
901
|
+
event: "owner-mutation-expired",
|
|
902
|
+
leaseMs,
|
|
903
|
+
mutationId,
|
|
904
|
+
operation,
|
|
905
|
+
pid: ownerMutationPid ?? ownerClientPid,
|
|
906
|
+
waitingFor: ownerMutationWaitingFor || "the operation to report completion"
|
|
907
|
+
}
|
|
908
|
+
|
|
909
|
+
process.stderr.write(`${JSON.stringify({at: new Date().toISOString(), ...diagnostic})}\n`)
|
|
910
|
+
if (!ownerMutationClient.destroyed) ownerMutationClient.write(`${JSON.stringify(diagnostic)}\n`)
|
|
911
|
+
broadcast(diagnostic)
|
|
912
|
+
ownerMutationExpired = {elapsedMs, leaseMs, mutationId, operation}
|
|
913
|
+
clearOwnerMutationLease()
|
|
914
|
+
}
|
|
915
|
+
|
|
723
916
|
/**
|
|
724
917
|
* @param {net.Socket} socket - Requesting client.
|
|
725
918
|
* @param {string} command - Command name.
|
|
@@ -740,10 +933,7 @@ function releaseClosedOwner(socket) {
|
|
|
740
933
|
rememberDisconnectedOwnerSuccessors(socket)
|
|
741
934
|
ownerClient = undefined
|
|
742
935
|
ownerClientPid = undefined
|
|
743
|
-
if (ownerMutationClient === socket)
|
|
744
|
-
ownerMutationClient = undefined
|
|
745
|
-
ownerMutationId = undefined
|
|
746
|
-
}
|
|
936
|
+
if (ownerMutationClient === socket) clearOwnerMutationLease()
|
|
747
937
|
if (replacementClient && replacementOwnerState) commitReplacement()
|
|
748
938
|
else {
|
|
749
939
|
if (incompleteRestartChild) {
|
package/src/process-memory.js
CHANGED
|
@@ -4,8 +4,14 @@ import fs from "node:fs"
|
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* @typedef {{command: string, pid: number, rssBytes: number | undefined}} ProcessGroupMember
|
|
7
|
+
* @typedef {{cpuMs: number | undefined, liveMemberCount: number, memberCount: number}} ProcessGroupSample
|
|
7
8
|
*/
|
|
8
9
|
|
|
10
|
+
// Linux reports /proc/<pid>/stat CPU time in clock ticks and keeps USER_HZ at 100 for
|
|
11
|
+
// userspace, so ticks * 10 / 1 is milliseconds. This is a monotonic per-process counter,
|
|
12
|
+
// not a rate: two reads of it measure whether a process did any work between them.
|
|
13
|
+
const CLOCK_TICKS_PER_MS = 10
|
|
14
|
+
|
|
9
15
|
/**
|
|
10
16
|
* Lists the members of a managed process group with each member's resident memory.
|
|
11
17
|
* Rollbridge spawns each process detached, so the spawned pid is the process-group
|
|
@@ -108,6 +114,51 @@ export function measureProcessGroupRssBytes(pgid) {
|
|
|
108
114
|
return measured.reduce((total, member) => total + (member.rssBytes ?? 0), 0)
|
|
109
115
|
}
|
|
110
116
|
|
|
117
|
+
/**
|
|
118
|
+
* Samples one managed process group for drain liveness: how much CPU its members have
|
|
119
|
+
* consumed in total, and how many of them can still run.
|
|
120
|
+
*
|
|
121
|
+
* CPU is the sum of every member's cumulative user+system time, defunct members included:
|
|
122
|
+
* a member that exits takes its consumed time out of the sum, so the difference between two
|
|
123
|
+
* samples is non-zero whenever the group changed at all — new work or a departing member.
|
|
124
|
+
* Only an exactly unchanged difference says the group did nothing between the two reads.
|
|
125
|
+
* @param {number} pgid - Process-group id (the detached spawn's pid).
|
|
126
|
+
* @param {string} [procPath] - Procfs root (overridable for deterministic tests).
|
|
127
|
+
* @returns {ProcessGroupSample | undefined} The sample, or undefined only when procfs cannot be
|
|
128
|
+
* read at all. A group with no members left is a sample of zero, not a missing measurement.
|
|
129
|
+
*/
|
|
130
|
+
export function sampleProcessGroup(pgid, procPath = "/proc") {
|
|
131
|
+
/** @type {string[]} */
|
|
132
|
+
let entries
|
|
133
|
+
|
|
134
|
+
try {
|
|
135
|
+
entries = fs.readdirSync(procPath)
|
|
136
|
+
} catch {
|
|
137
|
+
return undefined
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
let cpuMs = 0
|
|
141
|
+
let liveMemberCount = 0
|
|
142
|
+
let memberCount = 0
|
|
143
|
+
let measurable = true
|
|
144
|
+
|
|
145
|
+
for (const entry of entries) {
|
|
146
|
+
if (!/^\d+$/.test(entry)) continue
|
|
147
|
+
|
|
148
|
+
const stat = processStat(entry, procPath)
|
|
149
|
+
|
|
150
|
+
if (stat?.pgrp !== pgid) continue
|
|
151
|
+
|
|
152
|
+
memberCount += 1
|
|
153
|
+
if (stat.state !== "Z" && stat.state !== "X") liveMemberCount += 1
|
|
154
|
+
|
|
155
|
+
if (stat.cpuMs === undefined) measurable = false
|
|
156
|
+
else cpuMs += stat.cpuMs
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
return {cpuMs: measurable ? cpuMs : undefined, liveMemberCount, memberCount}
|
|
160
|
+
}
|
|
161
|
+
|
|
111
162
|
/**
|
|
112
163
|
* @param {string} pid - Process id.
|
|
113
164
|
* @returns {string} The process command name (`/proc/<pid>/comm`), or "" when unavailable.
|
|
@@ -131,7 +182,7 @@ function processGroupId(pid) {
|
|
|
131
182
|
/**
|
|
132
183
|
* @param {string} pid - Process id.
|
|
133
184
|
* @param {string} procPath - Procfs root.
|
|
134
|
-
* @returns {{pgrp: number, state: string} | undefined} Parsed process state and
|
|
185
|
+
* @returns {{cpuMs: number | undefined, pgrp: number, state: string} | undefined} Parsed process state, group, and cumulative CPU.
|
|
135
186
|
*/
|
|
136
187
|
function processStat(pid, procPath) {
|
|
137
188
|
let stat
|
|
@@ -151,7 +202,16 @@ function processStat(pid, procPath) {
|
|
|
151
202
|
const fields = stat.slice(commEnd + 2).split(" ")
|
|
152
203
|
const pgrp = Number(fields[2])
|
|
153
204
|
|
|
154
|
-
|
|
205
|
+
if (!Number.isInteger(pgrp)) return undefined
|
|
206
|
+
|
|
207
|
+
// utime and stime are fields 14 and 15 of stat, i.e. index 11 and 12 after the comm field.
|
|
208
|
+
const utimeTicks = Number(fields[11])
|
|
209
|
+
const stimeTicks = Number(fields[12])
|
|
210
|
+
const cpuMs = Number.isFinite(utimeTicks) && Number.isFinite(stimeTicks)
|
|
211
|
+
? (utimeTicks + stimeTicks) * CLOCK_TICKS_PER_MS
|
|
212
|
+
: undefined
|
|
213
|
+
|
|
214
|
+
return {cpuMs, pgrp, state: fields[0]}
|
|
155
215
|
}
|
|
156
216
|
|
|
157
217
|
/**
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* @typedef {import("./config.js").ProcessConfig} ProcessConfig
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
const POLICY_START_RANK = /** @type {Record<ProcessConfig["policy"], number>} */ ({companion: 1, proxied: 2, service: 0, singleton: 3})
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Order in which Rollbridge starts processes, which is also the only order a `dependsOn`
|
|
11
|
+
* edge may point backwards in: daemon-wide persistent services are started before every
|
|
12
|
+
* release generation, then handoff services, companions, and the proxied process last.
|
|
13
|
+
* Config order is kept inside each group.
|
|
14
|
+
* @param {ProcessConfig[]} processes - Normalized processes.
|
|
15
|
+
* @param {{includePersistentServices?: boolean}} [options] - Whether to include the
|
|
16
|
+
* daemon-wide persistent services that a release group never starts itself.
|
|
17
|
+
* @returns {ProcessConfig[]} Ordered process configs.
|
|
18
|
+
*/
|
|
19
|
+
export function processStartOrder(processes, {includePersistentServices = false} = {}) {
|
|
20
|
+
const persistentServices = includePersistentServices
|
|
21
|
+
? processes.filter((processConfig) => processConfig.policy === "service" && processConfig.deployStrategy === "persistent")
|
|
22
|
+
: []
|
|
23
|
+
const generationProcesses = processes.filter((processConfig) => processConfig.policy !== "singleton" && (processConfig.policy !== "service" || processConfig.deployStrategy === "handoff"))
|
|
24
|
+
|
|
25
|
+
return [...persistentServices, ...generationProcesses.sort((left, right) => POLICY_START_RANK[left.policy] - POLICY_START_RANK[right.policy])]
|
|
26
|
+
}
|