rollbridge 0.1.72 → 0.1.74
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +63 -10
- package/TODO.md +13 -0
- package/changelog.d/20261025-guardian-re-register-dead-generation.md +13 -0
- package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +1 -0
- package/changelog.d/20261104-compose-service-units.md +21 -0
- package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +20 -0
- package/changelog.d/20261106-release-dependency-boot-budgets.md +32 -0
- package/changelog.d/20261109-path-filtered-builds.md +11 -0
- package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +26 -0
- package/changelog.d/20261111-contention-sensitive-daemon-specs.md +27 -0
- package/changelog.d/20261113-bounded-nonblocking-retirement.md +5 -0
- package/changelog.d/20261115-bound-draining-release-groups.md +62 -0
- package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +27 -0
- package/changelog.d/20261115-wedged-generation-escalation.md +44 -0
- package/changelog.d/20261116-bounded-explicit-stop-retirement.md +30 -0
- package/changelog.d/20261116-config-json-canonical-shape.md +11 -0
- package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +29 -0
- package/changelog.d/20261116-guardian-version-handoff.md +37 -0
- package/changelog.d/20261116-reclaim-dead-owner-mutation.md +7 -0
- package/changelog.d/20261117-activate-before-incumbent-retirement.md +1 -0
- package/changelog.d/20261117-activation-path-drain-deadline.md +35 -0
- package/changelog.d/20261117-stop-companion-after-retired-dependency.md +7 -0
- package/changelog.d/20261118-incumbent-retirement-authority-guard.md +1 -0
- package/changelog.d/20261118-owner-retirement-no-guardian-bound.md +19 -0
- package/changelog.d/20261118-shutdown-retirement-bound-restored.md +31 -0
- package/docs/cli.md +128 -3
- package/docs/compose-services.md +222 -0
- package/docs/config.md +177 -6
- package/docs/troubleshooting.md +97 -6
- package/docs/velocious.md +1 -1
- package/docs/workers.md +11 -0
- package/examples/tensorbuzz.com.js +3 -2
- package/package.json +1 -1
- package/src/cli.js +68 -2
- package/src/compose-unit.js +725 -0
- package/src/config.js +346 -8
- package/src/daemon.js +410 -73
- package/src/doctor.js +74 -0
- package/src/drain-policy.js +252 -0
- package/src/guardian-client.js +67 -8
- package/src/health.js +99 -8
- package/src/managed-process.js +197 -16
- package/src/process-guardian.js +212 -22
- package/src/process-memory.js +62 -2
- package/src/process-order.js +26 -0
- package/src/release-group.js +435 -37
- package/tensorbuzz.yml +20 -1
- package/test/completion.test.js +5 -2
- package/test/compose-config.test.js +160 -0
- package/test/compose-docker.test.js +297 -0
- package/test/compose-unit.test.js +415 -0
- package/test/config-validation.test.js +135 -5
- package/test/drain-policy.test.js +328 -0
- package/test/fixtures/deferred-boot-app.js +106 -0
- package/test/fixtures/gated-app.js +50 -0
- package/test/guardian-client.test.js +119 -5
- package/test/health.test.js +74 -1
- package/test/managed-process.test.js +260 -1
- package/test/owner-recovery.test.js +125 -21
- package/test/owner-replacement.test.js +175 -14
- package/test/process-memory.test.js +47 -1
- package/test/release-group.test.js +408 -2
- package/test/rollbridge.test.js +774 -72
- package/test/shutdown-completion.test.js +119 -0
- package/test/shutdown-retirement-bound.test.js +162 -0
- package/test/support/process.js +3 -1
- package/test/tensorbuzz-config.test.js +228 -0
package/src/process-guardian.js
CHANGED
|
@@ -4,14 +4,21 @@ import fsSync from "node:fs"
|
|
|
4
4
|
import fs from "node:fs/promises"
|
|
5
5
|
import net from "node:net"
|
|
6
6
|
import crypto from "node:crypto"
|
|
7
|
+
import {createRequire} from "node:module"
|
|
7
8
|
import {spawn} from "node:child_process"
|
|
8
9
|
import {isDeepStrictEqual} from "node:util"
|
|
9
10
|
import path from "node:path"
|
|
10
11
|
import GuardianClient from "./guardian-client.js"
|
|
11
|
-
import ManagedProcess from "./managed-process.js"
|
|
12
|
+
import ManagedProcess, {childRecordIsGone} from "./managed-process.js"
|
|
12
13
|
import {writeState} from "./state-store.js"
|
|
13
14
|
|
|
14
15
|
const OWNER_RESTART_RETRY_MS = 1000
|
|
16
|
+
const commonJsRequire = createRequire(import.meta.url)
|
|
17
|
+
const GUARDIAN_VERSION = /** @type {{version: string}} */ (commonJsRequire("../package.json")).version
|
|
18
|
+
// An owner mutation is deploy/stop/restart/rollback/recovery work. A retained
|
|
19
|
+
// generation may legitimately drain for hours, but a mutation that holds the
|
|
20
|
+
// fence that long is stuck, so the lease is bounded and released on expiry.
|
|
21
|
+
const OWNER_MUTATION_LEASE_MS = 900000
|
|
15
22
|
|
|
16
23
|
/** @typedef {import("node:child_process").ChildProcess["signalCode"]} ChildExitSignal */
|
|
17
24
|
|
|
@@ -24,7 +31,7 @@ const OWNER_RESTART_RETRY_MS = 1000
|
|
|
24
31
|
* @property {number} id - Request id.
|
|
25
32
|
* @property {string} [key] - Stable process key.
|
|
26
33
|
* @property {boolean} [localSource] - Whether the retired daemon physically owns the published source.
|
|
27
|
-
* @property {{timeoutMs?: number}} [options] - Stop options.
|
|
34
|
+
* @property {{retirementTimeoutMs?: number, timeoutMs?: number}} [options] - Stop options.
|
|
28
35
|
* @property {string} [previousProvenance] - Expected current provenance.
|
|
29
36
|
* @property {string} [provenance] - New or registered provenance.
|
|
30
37
|
* @property {string} [releaseId] - Retained release identity.
|
|
@@ -32,7 +39,9 @@ const OWNER_RESTART_RETRY_MS = 1000
|
|
|
32
39
|
* @property {import("./json.js").JsonValue} [ownerState] - Private owner transfer state.
|
|
33
40
|
* @property {string} [replacementId] - Prepared replacement transaction id.
|
|
34
41
|
* @property {string} [mutationId] - Owner mutation lease id.
|
|
42
|
+
* @property {number} [mutationTimeoutMs] - Bound for the owner mutation lease.
|
|
35
43
|
* @property {string} [operation] - Owner mutation diagnostic name.
|
|
44
|
+
* @property {string} [waitingFor] - What the held owner mutation is blocked on.
|
|
36
45
|
* @property {number} [ownerPid] - Exact claimant process PID.
|
|
37
46
|
* @property {import("./json.js").JsonValue} [recoverySnapshot] - Public bridge recovery snapshot.
|
|
38
47
|
* @property {string} [statePath] - Exact public recovery state path.
|
|
@@ -103,6 +112,19 @@ const legacyKeys = legacyGuardian ? committedOwnerProcessKeys(ownerState) : new
|
|
|
103
112
|
let ownerMutationClient
|
|
104
113
|
/** @type {string | undefined} */
|
|
105
114
|
let ownerMutationId
|
|
115
|
+
/** @type {string | undefined} */
|
|
116
|
+
let ownerMutationOperation
|
|
117
|
+
/** @type {string | undefined} */
|
|
118
|
+
let ownerMutationWaitingFor
|
|
119
|
+
/** @type {number | undefined} */
|
|
120
|
+
let ownerMutationPid
|
|
121
|
+
/** @type {number | undefined} */
|
|
122
|
+
let ownerMutationStartedAt
|
|
123
|
+
let ownerMutationLeaseMs = OWNER_MUTATION_LEASE_MS
|
|
124
|
+
/** @type {ReturnType<typeof setTimeout> | undefined} */
|
|
125
|
+
let ownerMutationLeaseTimer
|
|
126
|
+
/** @type {{elapsedMs: number, leaseMs: number, mutationId: string, operation: string} | undefined} */
|
|
127
|
+
let ownerMutationExpired
|
|
106
128
|
/** @type {net.Socket | undefined} */
|
|
107
129
|
let retiringClient
|
|
108
130
|
/** @type {string | undefined} */
|
|
@@ -154,10 +176,7 @@ const server = net.createServer((socket) => {
|
|
|
154
176
|
clearTimeout(waiter.timer)
|
|
155
177
|
waiter.reject(new Error("Owner claimant disconnected"))
|
|
156
178
|
}
|
|
157
|
-
if (ownerMutationClient === socket
|
|
158
|
-
ownerMutationClient = undefined
|
|
159
|
-
ownerMutationId = undefined
|
|
160
|
-
}
|
|
179
|
+
if (ownerMutationClient === socket) clearOwnerMutationLease()
|
|
161
180
|
if (controlLessRetirement) {
|
|
162
181
|
const preCommitHandoff = replacementClient && ownerClient === socket && retiringReplacementId === replacementId
|
|
163
182
|
|
|
@@ -259,13 +278,14 @@ async function handleLine(socket, line) {
|
|
|
259
278
|
async function execute(request, socket) {
|
|
260
279
|
if (shuttingDown) throw new Error("Process guardian is shutting down")
|
|
261
280
|
|
|
262
|
-
if (request.command === "capabilities") return {daemonRecovery: 1, generationReactivation: 1}
|
|
281
|
+
if (request.command === "capabilities") return {daemonRecovery: 1, generationReactivation: 1, guardianVersion: GUARDIAN_VERSION}
|
|
263
282
|
|
|
264
283
|
if (request.command === "owner-replacement-capabilities") {
|
|
265
284
|
return {commands: ["commit-retired-owner-replacement"], protocol: "owner-replacement", version: 1}
|
|
266
285
|
}
|
|
267
286
|
|
|
268
287
|
if (request.command === "claim-owner") {
|
|
288
|
+
reclaimDeadOwnerMutation()
|
|
269
289
|
if (retirementFailed) throw new Error("Guardian owner replacement is fenced after incomplete retired-listener state transfer")
|
|
270
290
|
if (ownerState !== undefined && !isDeepStrictEqual(request.authority, ownerAuthority(ownerState))) {
|
|
271
291
|
const waitsForPreparedAuthority = Boolean(ownerClient && replacementOwnerState && isDeepStrictEqual(request.authority, replacementAuthority))
|
|
@@ -301,8 +321,7 @@ async function execute(request, socket) {
|
|
|
301
321
|
rememberDisconnectedOwnerSuccessors(socket)
|
|
302
322
|
ownerClient = undefined
|
|
303
323
|
ownerClientPid = undefined
|
|
304
|
-
|
|
305
|
-
ownerMutationId = undefined
|
|
324
|
+
clearOwnerMutationLease()
|
|
306
325
|
ownerRevision += 1
|
|
307
326
|
grantNextOwner()
|
|
308
327
|
return {retired: true}
|
|
@@ -394,29 +413,54 @@ async function execute(request, socket) {
|
|
|
394
413
|
return {committedReplacementId: committedReplacementId ?? null, ownerClaimed: Boolean(ownerClient), retirementFailed, retirementPending: Boolean(retiringClient), retirementReady: retiringListenerReady}
|
|
395
414
|
}
|
|
396
415
|
|
|
416
|
+
if (request.command === "guardian-status") return guardianStatus()
|
|
417
|
+
|
|
397
418
|
if (request.command === "begin-owner-mutation") {
|
|
398
419
|
requireOwner(socket, request.command)
|
|
420
|
+
reclaimDeadOwnerMutation()
|
|
399
421
|
if (replacementClient) throw new Error(`Owner mutation ${request.operation || "operation"} is fenced while an owner replacement is prepared`)
|
|
400
|
-
|
|
422
|
+
const leaseMs = request.mutationTimeoutMs === undefined ? OWNER_MUTATION_LEASE_MS : request.mutationTimeoutMs
|
|
423
|
+
|
|
424
|
+
if (!Number.isInteger(leaseMs) || leaseMs <= 0) throw new Error("Owner mutation lease bound must be a positive number of milliseconds")
|
|
425
|
+
if (ownerMutationClient) throw new Error(`Another owner mutation is already in progress: ${describeOwnerMutationLease()}; refused ${request.operation || "operation"}`)
|
|
401
426
|
ownerMutationClient = socket
|
|
402
427
|
ownerMutationId = crypto.randomUUID()
|
|
403
|
-
|
|
428
|
+
ownerMutationOperation = request.operation
|
|
429
|
+
ownerMutationWaitingFor = request.waitingFor
|
|
430
|
+
ownerMutationPid = request.ownerPid ?? ownerClientPid
|
|
431
|
+
ownerMutationStartedAt = Date.now()
|
|
432
|
+
ownerMutationLeaseMs = leaseMs
|
|
433
|
+
armOwnerMutationLease(ownerMutationId)
|
|
434
|
+
return {leaseMs, mutationId: ownerMutationId}
|
|
404
435
|
}
|
|
405
436
|
|
|
406
437
|
if (request.command === "end-owner-mutation") {
|
|
407
438
|
requireOwner(socket, request.command)
|
|
439
|
+
if (ownerMutationExpired && request.mutationId === ownerMutationExpired.mutationId) {
|
|
440
|
+
const expired = ownerMutationExpired
|
|
441
|
+
|
|
442
|
+
ownerMutationExpired = undefined
|
|
443
|
+
throw new Error(`Owner mutation ${expired.operation} exceeded its ${expired.leaseMs} ms lease and was released after ${expired.elapsedMs} ms; the mutation failed and its lease is gone`)
|
|
444
|
+
}
|
|
408
445
|
if (ownerMutationClient !== socket || request.mutationId !== ownerMutationId) throw new Error("Owner mutation lease does not match the active mutation")
|
|
409
|
-
|
|
410
|
-
ownerMutationId = undefined
|
|
446
|
+
clearOwnerMutationLease()
|
|
411
447
|
return {ended: true}
|
|
412
448
|
}
|
|
413
449
|
|
|
450
|
+
if (request.command === "report-owner-mutation-wait") {
|
|
451
|
+
requireOwner(socket, request.command)
|
|
452
|
+
if (ownerMutationClient !== socket || request.mutationId !== ownerMutationId) throw new Error("Owner mutation lease does not match the active mutation")
|
|
453
|
+
if (typeof request.waitingFor !== "string" || !request.waitingFor.trim()) throw new Error("Owner mutation wait report requires a waitingFor description")
|
|
454
|
+
ownerMutationWaitingFor = request.waitingFor.trim()
|
|
455
|
+
return {reported: true}
|
|
456
|
+
}
|
|
457
|
+
|
|
414
458
|
if (request.command === "prepare-owner-replacement") {
|
|
415
459
|
if (retirementFailed) throw new Error("Guardian owner replacement is fenced after incomplete retired-listener state transfer")
|
|
416
460
|
if (retiringClient) throw new Error("Guardian owner replacement cannot prepare while listener retirement is pending")
|
|
417
461
|
if (socket === ownerClient) throw new Error("Committed owner cannot prepare itself as a replacement")
|
|
418
462
|
if (replacementClient && replacementClient !== socket) throw new Error("Another owner replacement candidate is already prepared")
|
|
419
|
-
if (ownerMutationClient) throw new Error(
|
|
463
|
+
if (ownerMutationClient) throw new Error(`Owner replacement cannot prepare while an owner mutation is in progress: ${describeOwnerMutationLease()}`)
|
|
420
464
|
if (!ownerState) throw new Error("Committed owner has not published transferable state")
|
|
421
465
|
const currentAuthority = ownerAuthority(ownerState)
|
|
422
466
|
const resumesCommittedAuthority = !ownerClient && isDeepStrictEqual(currentAuthority, request.nextAuthority)
|
|
@@ -445,7 +489,7 @@ async function execute(request, socket) {
|
|
|
445
489
|
|
|
446
490
|
if (request.command === "stage-owner-replacement") {
|
|
447
491
|
requireReplacement(socket, request)
|
|
448
|
-
if (replacementRevision !== ownerRevision) throw new Error(
|
|
492
|
+
if (replacementRevision !== ownerRevision) throw new Error(`Guardian owner state changed after prepare (prepared revision ${replacementRevision}, current revision ${ownerRevision}); abort the stale candidate before preparing again`)
|
|
449
493
|
if (request.ownerState === undefined || !isDeepStrictEqual(ownerAuthority(request.ownerState), replacementAuthority)) {
|
|
450
494
|
throw new Error("Prepared replacement state does not match the requested authority")
|
|
451
495
|
}
|
|
@@ -576,7 +620,7 @@ async function execute(request, socket) {
|
|
|
576
620
|
requireOwner(socket, request.command)
|
|
577
621
|
shuttingDown = true
|
|
578
622
|
try {
|
|
579
|
-
const results = await Promise.allSettled([...processes.values()].map((entry) => entry.process.stop()))
|
|
623
|
+
const results = await Promise.allSettled([...processes.values()].map((entry) => entry.process.stop(request.options)))
|
|
580
624
|
const errors = results.filter((result) => result.status === "rejected").map((result) => result.reason)
|
|
581
625
|
|
|
582
626
|
if (errors.length > 0) throw new AggregateError(errors, `Guardian failed to stop ${errors.length} owned process${errors.length === 1 ? "" : "es"}: ${errors.map((error) => errorMessage(error instanceof Error ? error : String(error))).join("; ")}`)
|
|
@@ -613,8 +657,17 @@ async function execute(request, socket) {
|
|
|
613
657
|
const existing = processes.get(request.key)
|
|
614
658
|
|
|
615
659
|
if (existing) {
|
|
616
|
-
if (existing.provenance
|
|
617
|
-
|
|
660
|
+
if (existing.provenance === request.provenance) return existing.process.status()
|
|
661
|
+
// A provenance mismatch only fences a process that still exists. When the registered
|
|
662
|
+
// child is conclusively gone (stopped or failed with no live pid), the old record is
|
|
663
|
+
// residue of a dead generation, and demanding its provenance wedges every later deploy
|
|
664
|
+
// against a process that no longer exists (TensorBuzz production, STC-695). The owner
|
|
665
|
+
// may re-register the key under the new definition; live processes stay fail-closed.
|
|
666
|
+
// A group that outlived its tracked process is not gone: it still holds the key's work,
|
|
667
|
+
// so re-registering it would start a second copy of a generation that is still running.
|
|
668
|
+
if (!childRecordIsGone(existing.process.status())) throw new Error(`Guardian provenance mismatch for ${request.key}`)
|
|
669
|
+
requireOwner(socket, request.command)
|
|
670
|
+
processes.delete(request.key)
|
|
618
671
|
}
|
|
619
672
|
|
|
620
673
|
const recoversLegacyProcess = Boolean(legacyGuardian && legacyKeys.has(request.key))
|
|
@@ -705,6 +758,146 @@ async function execute(request, socket) {
|
|
|
705
758
|
return status
|
|
706
759
|
}
|
|
707
760
|
|
|
761
|
+
/**
|
|
762
|
+
* Names the mutation holding the owner-mutation fence: which mutation, which process,
|
|
763
|
+
* since when, and what it is blocked on. Refusals must be diagnosable without a live
|
|
764
|
+
* inspection of the daemon.
|
|
765
|
+
* @returns {string} Holder description used in refusal messages.
|
|
766
|
+
*/
|
|
767
|
+
function describeOwnerMutationLease() {
|
|
768
|
+
const startedAt = ownerMutationStartedAt ?? 0
|
|
769
|
+
const elapsedSeconds = Math.max(0, Math.round((Date.now() - startedAt) / 1000))
|
|
770
|
+
const holderPid = ownerMutationPid ?? ownerClientPid
|
|
771
|
+
|
|
772
|
+
return [
|
|
773
|
+
`owner mutation "${ownerMutationOperation || "unnamed operation"}"`,
|
|
774
|
+
`mutation ${ownerMutationId || "unknown"}`,
|
|
775
|
+
`held by process ${holderPid ?? "unknown"}`,
|
|
776
|
+
`since ${new Date(startedAt).toISOString()}`,
|
|
777
|
+
`${elapsedSeconds}s elapsed`,
|
|
778
|
+
`waiting for ${ownerMutationWaitingFor || "the operation to report completion"}`
|
|
779
|
+
].join("; ")
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
/**
|
|
783
|
+
* Reports the currently held owner-mutation lease.
|
|
784
|
+
* @returns {{elapsedMs: number, leaseMs: number, mutationId: string, operation: string, pid: number | null, startedAt: string, waitingFor: string} | null} Active lease, if any.
|
|
785
|
+
*/
|
|
786
|
+
function activeOwnerMutationStatus() {
|
|
787
|
+
if (!ownerMutationClient || !ownerMutationId) return null
|
|
788
|
+
|
|
789
|
+
const startedAt = ownerMutationStartedAt ?? Date.now()
|
|
790
|
+
|
|
791
|
+
return {
|
|
792
|
+
elapsedMs: Math.max(0, Date.now() - startedAt),
|
|
793
|
+
leaseMs: ownerMutationLeaseMs,
|
|
794
|
+
mutationId: ownerMutationId,
|
|
795
|
+
operation: ownerMutationOperation || "unnamed operation",
|
|
796
|
+
pid: ownerMutationPid ?? ownerClientPid ?? null,
|
|
797
|
+
startedAt: new Date(startedAt).toISOString(),
|
|
798
|
+
waitingFor: ownerMutationWaitingFor || "the operation to report completion"
|
|
799
|
+
}
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
/**
|
|
803
|
+
* Reports guardian identity and owner-mutation diagnostics.
|
|
804
|
+
* @returns {{guardianPid: number, ownerMutation: ReturnType<typeof activeOwnerMutationStatus>, version: string}} Guardian diagnostics.
|
|
805
|
+
*/
|
|
806
|
+
function guardianStatus() {
|
|
807
|
+
return {guardianPid: process.pid, ownerMutation: activeOwnerMutationStatus(), version: GUARDIAN_VERSION}
|
|
808
|
+
}
|
|
809
|
+
|
|
810
|
+
/** @returns {boolean} Whether the recorded owner-mutation holder is no longer alive. */
|
|
811
|
+
function reclaimDeadOwnerMutation() {
|
|
812
|
+
if (!ownerMutationClient || !ownerMutationPid || isProcessAlive(ownerMutationPid)) return false
|
|
813
|
+
|
|
814
|
+
const mutation = activeOwnerMutationStatus()
|
|
815
|
+
const deadOwnerSocket = ownerClient === ownerMutationClient && ownerClientPid === ownerMutationPid ? ownerClient : undefined
|
|
816
|
+
const diagnostic = {
|
|
817
|
+
code: "OWNER_MUTATION_RECLAIMED",
|
|
818
|
+
event: "owner-mutation-reclaimed",
|
|
819
|
+
mutationId: ownerMutationId,
|
|
820
|
+
operation: mutation?.operation || "unnamed operation",
|
|
821
|
+
pid: ownerMutationPid,
|
|
822
|
+
reason: "owner mutation holder process is no longer alive"
|
|
823
|
+
}
|
|
824
|
+
|
|
825
|
+
process.stderr.write(`${JSON.stringify({at: new Date().toISOString(), ...diagnostic})}\n`)
|
|
826
|
+
broadcast(diagnostic)
|
|
827
|
+
clearOwnerMutationLease()
|
|
828
|
+
if (deadOwnerSocket) releaseClosedOwner(deadOwnerSocket)
|
|
829
|
+
return true
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
/**
|
|
833
|
+
* Checks whether a process still exists without signaling it.
|
|
834
|
+
* @param {number} pid - Process id.
|
|
835
|
+
* @returns {boolean} Whether the process exists.
|
|
836
|
+
*/
|
|
837
|
+
function isProcessAlive(pid) {
|
|
838
|
+
try {
|
|
839
|
+
process.kill(pid, 0)
|
|
840
|
+
return true
|
|
841
|
+
} catch (error) {
|
|
842
|
+
return Boolean(error && typeof error === "object" && "code" in error && error.code === "EPERM")
|
|
843
|
+
}
|
|
844
|
+
}
|
|
845
|
+
|
|
846
|
+
/** @returns {void} */
|
|
847
|
+
function clearOwnerMutationLease() {
|
|
848
|
+
if (ownerMutationLeaseTimer) clearTimeout(ownerMutationLeaseTimer)
|
|
849
|
+
ownerMutationLeaseTimer = undefined
|
|
850
|
+
ownerMutationClient = undefined
|
|
851
|
+
ownerMutationId = undefined
|
|
852
|
+
ownerMutationOperation = undefined
|
|
853
|
+
ownerMutationWaitingFor = undefined
|
|
854
|
+
ownerMutationPid = undefined
|
|
855
|
+
ownerMutationStartedAt = undefined
|
|
856
|
+
ownerMutationLeaseMs = OWNER_MUTATION_LEASE_MS
|
|
857
|
+
}
|
|
858
|
+
|
|
859
|
+
/**
|
|
860
|
+
* Arms the lease bound for one mutation.
|
|
861
|
+
* @param {string} mutationId - Lease id the timer belongs to.
|
|
862
|
+
* @returns {void}
|
|
863
|
+
*/
|
|
864
|
+
function armOwnerMutationLease(mutationId) {
|
|
865
|
+
if (ownerMutationLeaseTimer) clearTimeout(ownerMutationLeaseTimer)
|
|
866
|
+
ownerMutationLeaseTimer = setTimeout(() => expireOwnerMutationLease(mutationId), ownerMutationLeaseMs)
|
|
867
|
+
ownerMutationLeaseTimer.unref()
|
|
868
|
+
}
|
|
869
|
+
|
|
870
|
+
/**
|
|
871
|
+
* A mutation that outruns its lease is reported and released instead of holding the fence
|
|
872
|
+
* forever, so a stuck mutation cannot lock out every later deploy. The holder learns the
|
|
873
|
+
* lease is gone when it tries to end it.
|
|
874
|
+
* @param {string} mutationId - Lease id the armed timer belongs to.
|
|
875
|
+
* @returns {void}
|
|
876
|
+
*/
|
|
877
|
+
function expireOwnerMutationLease(mutationId) {
|
|
878
|
+
ownerMutationLeaseTimer = undefined
|
|
879
|
+
if (!ownerMutationClient || ownerMutationId !== mutationId) return
|
|
880
|
+
const operation = ownerMutationOperation || "unnamed operation"
|
|
881
|
+
const leaseMs = ownerMutationLeaseMs
|
|
882
|
+
const elapsedMs = Math.max(0, Date.now() - (ownerMutationStartedAt ?? 0))
|
|
883
|
+
const diagnostic = {
|
|
884
|
+
code: "OWNER_MUTATION_LEASE_EXPIRED",
|
|
885
|
+
elapsedMs,
|
|
886
|
+
event: "owner-mutation-expired",
|
|
887
|
+
leaseMs,
|
|
888
|
+
mutationId,
|
|
889
|
+
operation,
|
|
890
|
+
pid: ownerMutationPid ?? ownerClientPid,
|
|
891
|
+
waitingFor: ownerMutationWaitingFor || "the operation to report completion"
|
|
892
|
+
}
|
|
893
|
+
|
|
894
|
+
process.stderr.write(`${JSON.stringify({at: new Date().toISOString(), ...diagnostic})}\n`)
|
|
895
|
+
if (!ownerMutationClient.destroyed) ownerMutationClient.write(`${JSON.stringify(diagnostic)}\n`)
|
|
896
|
+
broadcast(diagnostic)
|
|
897
|
+
ownerMutationExpired = {elapsedMs, leaseMs, mutationId, operation}
|
|
898
|
+
clearOwnerMutationLease()
|
|
899
|
+
}
|
|
900
|
+
|
|
708
901
|
/**
|
|
709
902
|
* @param {net.Socket} socket - Requesting client.
|
|
710
903
|
* @param {string} command - Command name.
|
|
@@ -725,10 +918,7 @@ function releaseClosedOwner(socket) {
|
|
|
725
918
|
rememberDisconnectedOwnerSuccessors(socket)
|
|
726
919
|
ownerClient = undefined
|
|
727
920
|
ownerClientPid = undefined
|
|
728
|
-
if (ownerMutationClient === socket)
|
|
729
|
-
ownerMutationClient = undefined
|
|
730
|
-
ownerMutationId = undefined
|
|
731
|
-
}
|
|
921
|
+
if (ownerMutationClient === socket) clearOwnerMutationLease()
|
|
732
922
|
if (replacementClient && replacementOwnerState) commitReplacement()
|
|
733
923
|
else {
|
|
734
924
|
if (incompleteRestartChild) {
|
package/src/process-memory.js
CHANGED
|
@@ -4,8 +4,14 @@ import fs from "node:fs"
|
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* @typedef {{command: string, pid: number, rssBytes: number | undefined}} ProcessGroupMember
|
|
7
|
+
* @typedef {{cpuMs: number | undefined, liveMemberCount: number, memberCount: number}} ProcessGroupSample
|
|
7
8
|
*/
|
|
8
9
|
|
|
10
|
+
// Linux reports /proc/<pid>/stat CPU time in clock ticks and keeps USER_HZ at 100 for
|
|
11
|
+
// userspace, so ticks * 10 / 1 is milliseconds. This is a monotonic per-process counter,
|
|
12
|
+
// not a rate: two reads of it measure whether a process did any work between them.
|
|
13
|
+
const CLOCK_TICKS_PER_MS = 10
|
|
14
|
+
|
|
9
15
|
/**
|
|
10
16
|
* Lists the members of a managed process group with each member's resident memory.
|
|
11
17
|
* Rollbridge spawns each process detached, so the spawned pid is the process-group
|
|
@@ -83,6 +89,51 @@ export function measureProcessGroupRssBytes(pgid) {
|
|
|
83
89
|
return measured.reduce((total, member) => total + (member.rssBytes ?? 0), 0)
|
|
84
90
|
}
|
|
85
91
|
|
|
92
|
+
/**
|
|
93
|
+
* Samples one managed process group for drain liveness: how much CPU its members have
|
|
94
|
+
* consumed in total, and how many of them can still run.
|
|
95
|
+
*
|
|
96
|
+
* CPU is the sum of every member's cumulative user+system time, defunct members included:
|
|
97
|
+
* a member that exits takes its consumed time out of the sum, so the difference between two
|
|
98
|
+
* samples is non-zero whenever the group changed at all — new work or a departing member.
|
|
99
|
+
* Only an exactly unchanged difference says the group did nothing between the two reads.
|
|
100
|
+
* @param {number} pgid - Process-group id (the detached spawn's pid).
|
|
101
|
+
* @param {string} [procPath] - Procfs root (overridable for deterministic tests).
|
|
102
|
+
* @returns {ProcessGroupSample | undefined} The sample, or undefined only when procfs cannot be
|
|
103
|
+
* read at all. A group with no members left is a sample of zero, not a missing measurement.
|
|
104
|
+
*/
|
|
105
|
+
export function sampleProcessGroup(pgid, procPath = "/proc") {
|
|
106
|
+
/** @type {string[]} */
|
|
107
|
+
let entries
|
|
108
|
+
|
|
109
|
+
try {
|
|
110
|
+
entries = fs.readdirSync(procPath)
|
|
111
|
+
} catch {
|
|
112
|
+
return undefined
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
let cpuMs = 0
|
|
116
|
+
let liveMemberCount = 0
|
|
117
|
+
let memberCount = 0
|
|
118
|
+
let measurable = true
|
|
119
|
+
|
|
120
|
+
for (const entry of entries) {
|
|
121
|
+
if (!/^\d+$/.test(entry)) continue
|
|
122
|
+
|
|
123
|
+
const stat = processStat(entry, procPath)
|
|
124
|
+
|
|
125
|
+
if (stat?.pgrp !== pgid) continue
|
|
126
|
+
|
|
127
|
+
memberCount += 1
|
|
128
|
+
if (stat.state !== "Z" && stat.state !== "X") liveMemberCount += 1
|
|
129
|
+
|
|
130
|
+
if (stat.cpuMs === undefined) measurable = false
|
|
131
|
+
else cpuMs += stat.cpuMs
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
return {cpuMs: measurable ? cpuMs : undefined, liveMemberCount, memberCount}
|
|
135
|
+
}
|
|
136
|
+
|
|
86
137
|
/**
|
|
87
138
|
* @param {string} pid - Process id.
|
|
88
139
|
* @returns {string} The process command name (`/proc/<pid>/comm`), or "" when unavailable.
|
|
@@ -106,7 +157,7 @@ function processGroupId(pid) {
|
|
|
106
157
|
/**
|
|
107
158
|
* @param {string} pid - Process id.
|
|
108
159
|
* @param {string} procPath - Procfs root.
|
|
109
|
-
* @returns {{pgrp: number, state: string} | undefined} Parsed process state and
|
|
160
|
+
* @returns {{cpuMs: number | undefined, pgrp: number, state: string} | undefined} Parsed process state, group, and cumulative CPU.
|
|
110
161
|
*/
|
|
111
162
|
function processStat(pid, procPath) {
|
|
112
163
|
let stat
|
|
@@ -126,7 +177,16 @@ function processStat(pid, procPath) {
|
|
|
126
177
|
const fields = stat.slice(commEnd + 2).split(" ")
|
|
127
178
|
const pgrp = Number(fields[2])
|
|
128
179
|
|
|
129
|
-
|
|
180
|
+
if (!Number.isInteger(pgrp)) return undefined
|
|
181
|
+
|
|
182
|
+
// utime and stime are fields 14 and 15 of stat, i.e. index 11 and 12 after the comm field.
|
|
183
|
+
const utimeTicks = Number(fields[11])
|
|
184
|
+
const stimeTicks = Number(fields[12])
|
|
185
|
+
const cpuMs = Number.isFinite(utimeTicks) && Number.isFinite(stimeTicks)
|
|
186
|
+
? (utimeTicks + stimeTicks) * CLOCK_TICKS_PER_MS
|
|
187
|
+
: undefined
|
|
188
|
+
|
|
189
|
+
return {cpuMs, pgrp, state: fields[0]}
|
|
130
190
|
}
|
|
131
191
|
|
|
132
192
|
/**
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* @typedef {import("./config.js").ProcessConfig} ProcessConfig
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
const POLICY_START_RANK = /** @type {Record<ProcessConfig["policy"], number>} */ ({companion: 1, proxied: 2, service: 0, singleton: 3})
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Order in which Rollbridge starts processes, which is also the only order a `dependsOn`
|
|
11
|
+
* edge may point backwards in: daemon-wide persistent services are started before every
|
|
12
|
+
* release generation, then handoff services, companions, and the proxied process last.
|
|
13
|
+
* Config order is kept inside each group.
|
|
14
|
+
* @param {ProcessConfig[]} processes - Normalized processes.
|
|
15
|
+
* @param {{includePersistentServices?: boolean}} [options] - Whether to include the
|
|
16
|
+
* daemon-wide persistent services that a release group never starts itself.
|
|
17
|
+
* @returns {ProcessConfig[]} Ordered process configs.
|
|
18
|
+
*/
|
|
19
|
+
export function processStartOrder(processes, {includePersistentServices = false} = {}) {
|
|
20
|
+
const persistentServices = includePersistentServices
|
|
21
|
+
? processes.filter((processConfig) => processConfig.policy === "service" && processConfig.deployStrategy === "persistent")
|
|
22
|
+
: []
|
|
23
|
+
const generationProcesses = processes.filter((processConfig) => processConfig.policy !== "singleton" && (processConfig.policy !== "service" || processConfig.deployStrategy === "handoff"))
|
|
24
|
+
|
|
25
|
+
return [...persistentServices, ...generationProcesses.sort((left, right) => POLICY_START_RANK[left.policy] - POLICY_START_RANK[right.policy])]
|
|
26
|
+
}
|