rollbridge 0.1.69 → 0.1.71
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -53
- package/TODO.md +0 -12
- package/docs/cli.md +3 -128
- package/docs/config.md +6 -177
- package/docs/troubleshooting.md +6 -97
- package/docs/velocious.md +1 -1
- package/docs/workers.md +0 -11
- package/examples/tensorbuzz.com.js +2 -3
- package/package.json +1 -1
- package/src/cli.js +2 -68
- package/src/config.js +8 -347
- package/src/daemon.js +69 -370
- package/src/doctor.js +0 -74
- package/src/guardian-client.js +8 -67
- package/src/health.js +8 -99
- package/src/managed-process.js +16 -197
- package/src/process-guardian.js +21 -211
- package/src/process-memory.js +2 -62
- package/src/release-group.js +37 -427
- package/tensorbuzz.yml +1 -20
- package/test/completion.test.js +2 -5
- package/test/config-validation.test.js +5 -135
- package/test/guardian-client.test.js +4 -118
- package/test/health.test.js +1 -74
- package/test/managed-process.test.js +1 -260
- package/test/owner-recovery.test.js +28 -132
- package/test/owner-replacement.test.js +11 -172
- package/test/process-memory.test.js +1 -47
- package/test/release-group.test.js +2 -408
- package/test/rollbridge.test.js +82 -766
- package/test/shutdown-completion.test.js +0 -52
- package/test/support/process.js +1 -3
- package/changelog.d/20261025-guardian-re-register-dead-generation.md +0 -13
- package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +0 -1
- package/changelog.d/20261104-compose-service-units.md +0 -21
- package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +0 -20
- package/changelog.d/20261106-release-dependency-boot-budgets.md +0 -32
- package/changelog.d/20261109-path-filtered-builds.md +0 -11
- package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +0 -26
- package/changelog.d/20261111-contention-sensitive-daemon-specs.md +0 -27
- package/changelog.d/20261113-bounded-nonblocking-retirement.md +0 -5
- package/changelog.d/20261115-bound-draining-release-groups.md +0 -62
- package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +0 -27
- package/changelog.d/20261115-wedged-generation-escalation.md +0 -44
- package/changelog.d/20261116-bounded-explicit-stop-retirement.md +0 -30
- package/changelog.d/20261116-config-json-canonical-shape.md +0 -11
- package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +0 -29
- package/changelog.d/20261116-guardian-version-handoff.md +0 -37
- package/changelog.d/20261116-reclaim-dead-owner-mutation.md +0 -7
- package/changelog.d/20261117-activation-path-drain-deadline.md +0 -35
- package/changelog.d/20261117-stop-companion-after-retired-dependency.md +0 -7
- package/docs/compose-services.md +0 -222
- package/src/compose-unit.js +0 -725
- package/src/drain-policy.js +0 -252
- package/src/process-order.js +0 -26
- package/test/compose-config.test.js +0 -160
- package/test/compose-docker.test.js +0 -297
- package/test/compose-unit.test.js +0 -415
- package/test/drain-policy.test.js +0 -328
- package/test/fixtures/deferred-boot-app.js +0 -106
- package/test/fixtures/gated-app.js +0 -50
- package/test/tensorbuzz-config.test.js +0 -228
package/src/process-guardian.js
CHANGED
|
@@ -4,21 +4,14 @@ import fsSync from "node:fs"
|
|
|
4
4
|
import fs from "node:fs/promises"
|
|
5
5
|
import net from "node:net"
|
|
6
6
|
import crypto from "node:crypto"
|
|
7
|
-
import {createRequire} from "node:module"
|
|
8
7
|
import {spawn} from "node:child_process"
|
|
9
8
|
import {isDeepStrictEqual} from "node:util"
|
|
10
9
|
import path from "node:path"
|
|
11
10
|
import GuardianClient from "./guardian-client.js"
|
|
12
|
-
import ManagedProcess
|
|
11
|
+
import ManagedProcess from "./managed-process.js"
|
|
13
12
|
import {writeState} from "./state-store.js"
|
|
14
13
|
|
|
15
14
|
const OWNER_RESTART_RETRY_MS = 1000
|
|
16
|
-
const commonJsRequire = createRequire(import.meta.url)
|
|
17
|
-
const GUARDIAN_VERSION = /** @type {{version: string}} */ (commonJsRequire("../package.json")).version
|
|
18
|
-
// An owner mutation is deploy/stop/restart/rollback/recovery work. A retained
|
|
19
|
-
// generation may legitimately drain for hours, but a mutation that holds the
|
|
20
|
-
// fence that long is stuck, so the lease is bounded and released on expiry.
|
|
21
|
-
const OWNER_MUTATION_LEASE_MS = 900000
|
|
22
15
|
|
|
23
16
|
/** @typedef {import("node:child_process").ChildProcess["signalCode"]} ChildExitSignal */
|
|
24
17
|
|
|
@@ -31,7 +24,7 @@ const OWNER_MUTATION_LEASE_MS = 900000
|
|
|
31
24
|
* @property {number} id - Request id.
|
|
32
25
|
* @property {string} [key] - Stable process key.
|
|
33
26
|
* @property {boolean} [localSource] - Whether the retired daemon physically owns the published source.
|
|
34
|
-
* @property {{
|
|
27
|
+
* @property {{timeoutMs?: number}} [options] - Stop options.
|
|
35
28
|
* @property {string} [previousProvenance] - Expected current provenance.
|
|
36
29
|
* @property {string} [provenance] - New or registered provenance.
|
|
37
30
|
* @property {string} [releaseId] - Retained release identity.
|
|
@@ -39,9 +32,7 @@ const OWNER_MUTATION_LEASE_MS = 900000
|
|
|
39
32
|
* @property {import("./json.js").JsonValue} [ownerState] - Private owner transfer state.
|
|
40
33
|
* @property {string} [replacementId] - Prepared replacement transaction id.
|
|
41
34
|
* @property {string} [mutationId] - Owner mutation lease id.
|
|
42
|
-
* @property {number} [mutationTimeoutMs] - Bound for the owner mutation lease.
|
|
43
35
|
* @property {string} [operation] - Owner mutation diagnostic name.
|
|
44
|
-
* @property {string} [waitingFor] - What the held owner mutation is blocked on.
|
|
45
36
|
* @property {number} [ownerPid] - Exact claimant process PID.
|
|
46
37
|
* @property {import("./json.js").JsonValue} [recoverySnapshot] - Public bridge recovery snapshot.
|
|
47
38
|
* @property {string} [statePath] - Exact public recovery state path.
|
|
@@ -112,19 +103,6 @@ const legacyKeys = legacyGuardian ? committedOwnerProcessKeys(ownerState) : new
|
|
|
112
103
|
let ownerMutationClient
|
|
113
104
|
/** @type {string | undefined} */
|
|
114
105
|
let ownerMutationId
|
|
115
|
-
/** @type {string | undefined} */
|
|
116
|
-
let ownerMutationOperation
|
|
117
|
-
/** @type {string | undefined} */
|
|
118
|
-
let ownerMutationWaitingFor
|
|
119
|
-
/** @type {number | undefined} */
|
|
120
|
-
let ownerMutationPid
|
|
121
|
-
/** @type {number | undefined} */
|
|
122
|
-
let ownerMutationStartedAt
|
|
123
|
-
let ownerMutationLeaseMs = OWNER_MUTATION_LEASE_MS
|
|
124
|
-
/** @type {ReturnType<typeof setTimeout> | undefined} */
|
|
125
|
-
let ownerMutationLeaseTimer
|
|
126
|
-
/** @type {{elapsedMs: number, leaseMs: number, mutationId: string, operation: string} | undefined} */
|
|
127
|
-
let ownerMutationExpired
|
|
128
106
|
/** @type {net.Socket | undefined} */
|
|
129
107
|
let retiringClient
|
|
130
108
|
/** @type {string | undefined} */
|
|
@@ -176,7 +154,10 @@ const server = net.createServer((socket) => {
|
|
|
176
154
|
clearTimeout(waiter.timer)
|
|
177
155
|
waiter.reject(new Error("Owner claimant disconnected"))
|
|
178
156
|
}
|
|
179
|
-
if (ownerMutationClient === socket)
|
|
157
|
+
if (ownerMutationClient === socket && ownerClient !== socket) {
|
|
158
|
+
ownerMutationClient = undefined
|
|
159
|
+
ownerMutationId = undefined
|
|
160
|
+
}
|
|
180
161
|
if (controlLessRetirement) {
|
|
181
162
|
const preCommitHandoff = replacementClient && ownerClient === socket && retiringReplacementId === replacementId
|
|
182
163
|
|
|
@@ -278,14 +259,13 @@ async function handleLine(socket, line) {
|
|
|
278
259
|
async function execute(request, socket) {
|
|
279
260
|
if (shuttingDown) throw new Error("Process guardian is shutting down")
|
|
280
261
|
|
|
281
|
-
if (request.command === "capabilities") return {daemonRecovery: 1, generationReactivation: 1
|
|
262
|
+
if (request.command === "capabilities") return {daemonRecovery: 1, generationReactivation: 1}
|
|
282
263
|
|
|
283
264
|
if (request.command === "owner-replacement-capabilities") {
|
|
284
265
|
return {commands: ["commit-retired-owner-replacement"], protocol: "owner-replacement", version: 1}
|
|
285
266
|
}
|
|
286
267
|
|
|
287
268
|
if (request.command === "claim-owner") {
|
|
288
|
-
reclaimDeadOwnerMutation()
|
|
289
269
|
if (retirementFailed) throw new Error("Guardian owner replacement is fenced after incomplete retired-listener state transfer")
|
|
290
270
|
if (ownerState !== undefined && !isDeepStrictEqual(request.authority, ownerAuthority(ownerState))) {
|
|
291
271
|
const waitsForPreparedAuthority = Boolean(ownerClient && replacementOwnerState && isDeepStrictEqual(request.authority, replacementAuthority))
|
|
@@ -321,7 +301,8 @@ async function execute(request, socket) {
|
|
|
321
301
|
rememberDisconnectedOwnerSuccessors(socket)
|
|
322
302
|
ownerClient = undefined
|
|
323
303
|
ownerClientPid = undefined
|
|
324
|
-
|
|
304
|
+
ownerMutationClient = undefined
|
|
305
|
+
ownerMutationId = undefined
|
|
325
306
|
ownerRevision += 1
|
|
326
307
|
grantNextOwner()
|
|
327
308
|
return {retired: true}
|
|
@@ -413,54 +394,29 @@ async function execute(request, socket) {
|
|
|
413
394
|
return {committedReplacementId: committedReplacementId ?? null, ownerClaimed: Boolean(ownerClient), retirementFailed, retirementPending: Boolean(retiringClient), retirementReady: retiringListenerReady}
|
|
414
395
|
}
|
|
415
396
|
|
|
416
|
-
if (request.command === "guardian-status") return guardianStatus()
|
|
417
|
-
|
|
418
397
|
if (request.command === "begin-owner-mutation") {
|
|
419
398
|
requireOwner(socket, request.command)
|
|
420
|
-
reclaimDeadOwnerMutation()
|
|
421
399
|
if (replacementClient) throw new Error(`Owner mutation ${request.operation || "operation"} is fenced while an owner replacement is prepared`)
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
if (!Number.isInteger(leaseMs) || leaseMs <= 0) throw new Error("Owner mutation lease bound must be a positive number of milliseconds")
|
|
425
|
-
if (ownerMutationClient) throw new Error(`Another owner mutation is already in progress: ${describeOwnerMutationLease()}; refused ${request.operation || "operation"}`)
|
|
400
|
+
if (ownerMutationClient) throw new Error("Another owner mutation is already in progress")
|
|
426
401
|
ownerMutationClient = socket
|
|
427
402
|
ownerMutationId = crypto.randomUUID()
|
|
428
|
-
|
|
429
|
-
ownerMutationWaitingFor = request.waitingFor
|
|
430
|
-
ownerMutationPid = request.ownerPid ?? ownerClientPid
|
|
431
|
-
ownerMutationStartedAt = Date.now()
|
|
432
|
-
ownerMutationLeaseMs = leaseMs
|
|
433
|
-
armOwnerMutationLease(ownerMutationId)
|
|
434
|
-
return {leaseMs, mutationId: ownerMutationId}
|
|
403
|
+
return {mutationId: ownerMutationId}
|
|
435
404
|
}
|
|
436
405
|
|
|
437
406
|
if (request.command === "end-owner-mutation") {
|
|
438
407
|
requireOwner(socket, request.command)
|
|
439
|
-
if (ownerMutationExpired && request.mutationId === ownerMutationExpired.mutationId) {
|
|
440
|
-
const expired = ownerMutationExpired
|
|
441
|
-
|
|
442
|
-
ownerMutationExpired = undefined
|
|
443
|
-
throw new Error(`Owner mutation ${expired.operation} exceeded its ${expired.leaseMs} ms lease and was released after ${expired.elapsedMs} ms; the mutation failed and its lease is gone`)
|
|
444
|
-
}
|
|
445
408
|
if (ownerMutationClient !== socket || request.mutationId !== ownerMutationId) throw new Error("Owner mutation lease does not match the active mutation")
|
|
446
|
-
|
|
409
|
+
ownerMutationClient = undefined
|
|
410
|
+
ownerMutationId = undefined
|
|
447
411
|
return {ended: true}
|
|
448
412
|
}
|
|
449
413
|
|
|
450
|
-
if (request.command === "report-owner-mutation-wait") {
|
|
451
|
-
requireOwner(socket, request.command)
|
|
452
|
-
if (ownerMutationClient !== socket || request.mutationId !== ownerMutationId) throw new Error("Owner mutation lease does not match the active mutation")
|
|
453
|
-
if (typeof request.waitingFor !== "string" || !request.waitingFor.trim()) throw new Error("Owner mutation wait report requires a waitingFor description")
|
|
454
|
-
ownerMutationWaitingFor = request.waitingFor.trim()
|
|
455
|
-
return {reported: true}
|
|
456
|
-
}
|
|
457
|
-
|
|
458
414
|
if (request.command === "prepare-owner-replacement") {
|
|
459
415
|
if (retirementFailed) throw new Error("Guardian owner replacement is fenced after incomplete retired-listener state transfer")
|
|
460
416
|
if (retiringClient) throw new Error("Guardian owner replacement cannot prepare while listener retirement is pending")
|
|
461
417
|
if (socket === ownerClient) throw new Error("Committed owner cannot prepare itself as a replacement")
|
|
462
418
|
if (replacementClient && replacementClient !== socket) throw new Error("Another owner replacement candidate is already prepared")
|
|
463
|
-
if (ownerMutationClient) throw new Error(
|
|
419
|
+
if (ownerMutationClient) throw new Error("Owner replacement cannot prepare while an owner mutation is in progress")
|
|
464
420
|
if (!ownerState) throw new Error("Committed owner has not published transferable state")
|
|
465
421
|
const currentAuthority = ownerAuthority(ownerState)
|
|
466
422
|
const resumesCommittedAuthority = !ownerClient && isDeepStrictEqual(currentAuthority, request.nextAuthority)
|
|
@@ -620,7 +576,7 @@ async function execute(request, socket) {
|
|
|
620
576
|
requireOwner(socket, request.command)
|
|
621
577
|
shuttingDown = true
|
|
622
578
|
try {
|
|
623
|
-
const results = await Promise.allSettled([...processes.values()].map((entry) => entry.process.stop(
|
|
579
|
+
const results = await Promise.allSettled([...processes.values()].map((entry) => entry.process.stop()))
|
|
624
580
|
const errors = results.filter((result) => result.status === "rejected").map((result) => result.reason)
|
|
625
581
|
|
|
626
582
|
if (errors.length > 0) throw new AggregateError(errors, `Guardian failed to stop ${errors.length} owned process${errors.length === 1 ? "" : "es"}: ${errors.map((error) => errorMessage(error instanceof Error ? error : String(error))).join("; ")}`)
|
|
@@ -657,17 +613,8 @@ async function execute(request, socket) {
|
|
|
657
613
|
const existing = processes.get(request.key)
|
|
658
614
|
|
|
659
615
|
if (existing) {
|
|
660
|
-
if (existing.provenance
|
|
661
|
-
|
|
662
|
-
// child is conclusively gone (stopped or failed with no live pid), the old record is
|
|
663
|
-
// residue of a dead generation, and demanding its provenance wedges every later deploy
|
|
664
|
-
// against a process that no longer exists (TensorBuzz production, STC-695). The owner
|
|
665
|
-
// may re-register the key under the new definition; live processes stay fail-closed.
|
|
666
|
-
// A group that outlived its tracked process is not gone: it still holds the key's work,
|
|
667
|
-
// so re-registering it would start a second copy of a generation that is still running.
|
|
668
|
-
if (!childRecordIsGone(existing.process.status())) throw new Error(`Guardian provenance mismatch for ${request.key}`)
|
|
669
|
-
requireOwner(socket, request.command)
|
|
670
|
-
processes.delete(request.key)
|
|
616
|
+
if (existing.provenance !== request.provenance) throw new Error(`Guardian provenance mismatch for ${request.key}`)
|
|
617
|
+
return existing.process.status()
|
|
671
618
|
}
|
|
672
619
|
|
|
673
620
|
const recoversLegacyProcess = Boolean(legacyGuardian && legacyKeys.has(request.key))
|
|
@@ -758,146 +705,6 @@ async function execute(request, socket) {
|
|
|
758
705
|
return status
|
|
759
706
|
}
|
|
760
707
|
|
|
761
|
-
/**
|
|
762
|
-
* Names the mutation holding the owner-mutation fence: which mutation, which process,
|
|
763
|
-
* since when, and what it is blocked on. Refusals must be diagnosable without a live
|
|
764
|
-
* inspection of the daemon.
|
|
765
|
-
* @returns {string} Holder description used in refusal messages.
|
|
766
|
-
*/
|
|
767
|
-
function describeOwnerMutationLease() {
|
|
768
|
-
const startedAt = ownerMutationStartedAt ?? 0
|
|
769
|
-
const elapsedSeconds = Math.max(0, Math.round((Date.now() - startedAt) / 1000))
|
|
770
|
-
const holderPid = ownerMutationPid ?? ownerClientPid
|
|
771
|
-
|
|
772
|
-
return [
|
|
773
|
-
`owner mutation "${ownerMutationOperation || "unnamed operation"}"`,
|
|
774
|
-
`mutation ${ownerMutationId || "unknown"}`,
|
|
775
|
-
`held by process ${holderPid ?? "unknown"}`,
|
|
776
|
-
`since ${new Date(startedAt).toISOString()}`,
|
|
777
|
-
`${elapsedSeconds}s elapsed`,
|
|
778
|
-
`waiting for ${ownerMutationWaitingFor || "the operation to report completion"}`
|
|
779
|
-
].join("; ")
|
|
780
|
-
}
|
|
781
|
-
|
|
782
|
-
/**
|
|
783
|
-
* Reports the currently held owner-mutation lease.
|
|
784
|
-
* @returns {{elapsedMs: number, leaseMs: number, mutationId: string, operation: string, pid: number | null, startedAt: string, waitingFor: string} | null} Active lease, if any.
|
|
785
|
-
*/
|
|
786
|
-
function activeOwnerMutationStatus() {
|
|
787
|
-
if (!ownerMutationClient || !ownerMutationId) return null
|
|
788
|
-
|
|
789
|
-
const startedAt = ownerMutationStartedAt ?? Date.now()
|
|
790
|
-
|
|
791
|
-
return {
|
|
792
|
-
elapsedMs: Math.max(0, Date.now() - startedAt),
|
|
793
|
-
leaseMs: ownerMutationLeaseMs,
|
|
794
|
-
mutationId: ownerMutationId,
|
|
795
|
-
operation: ownerMutationOperation || "unnamed operation",
|
|
796
|
-
pid: ownerMutationPid ?? ownerClientPid ?? null,
|
|
797
|
-
startedAt: new Date(startedAt).toISOString(),
|
|
798
|
-
waitingFor: ownerMutationWaitingFor || "the operation to report completion"
|
|
799
|
-
}
|
|
800
|
-
}
|
|
801
|
-
|
|
802
|
-
/**
|
|
803
|
-
* Reports guardian identity and owner-mutation diagnostics.
|
|
804
|
-
* @returns {{guardianPid: number, ownerMutation: ReturnType<typeof activeOwnerMutationStatus>, version: string}} Guardian diagnostics.
|
|
805
|
-
*/
|
|
806
|
-
function guardianStatus() {
|
|
807
|
-
return {guardianPid: process.pid, ownerMutation: activeOwnerMutationStatus(), version: GUARDIAN_VERSION}
|
|
808
|
-
}
|
|
809
|
-
|
|
810
|
-
/** @returns {boolean} Whether the recorded owner-mutation holder is no longer alive. */
|
|
811
|
-
function reclaimDeadOwnerMutation() {
|
|
812
|
-
if (!ownerMutationClient || !ownerMutationPid || isProcessAlive(ownerMutationPid)) return false
|
|
813
|
-
|
|
814
|
-
const mutation = activeOwnerMutationStatus()
|
|
815
|
-
const deadOwnerSocket = ownerClient === ownerMutationClient && ownerClientPid === ownerMutationPid ? ownerClient : undefined
|
|
816
|
-
const diagnostic = {
|
|
817
|
-
code: "OWNER_MUTATION_RECLAIMED",
|
|
818
|
-
event: "owner-mutation-reclaimed",
|
|
819
|
-
mutationId: ownerMutationId,
|
|
820
|
-
operation: mutation?.operation || "unnamed operation",
|
|
821
|
-
pid: ownerMutationPid,
|
|
822
|
-
reason: "owner mutation holder process is no longer alive"
|
|
823
|
-
}
|
|
824
|
-
|
|
825
|
-
process.stderr.write(`${JSON.stringify({at: new Date().toISOString(), ...diagnostic})}\n`)
|
|
826
|
-
broadcast(diagnostic)
|
|
827
|
-
clearOwnerMutationLease()
|
|
828
|
-
if (deadOwnerSocket) releaseClosedOwner(deadOwnerSocket)
|
|
829
|
-
return true
|
|
830
|
-
}
|
|
831
|
-
|
|
832
|
-
/**
|
|
833
|
-
* Checks whether a process still exists without signaling it.
|
|
834
|
-
* @param {number} pid - Process id.
|
|
835
|
-
* @returns {boolean} Whether the process exists.
|
|
836
|
-
*/
|
|
837
|
-
function isProcessAlive(pid) {
|
|
838
|
-
try {
|
|
839
|
-
process.kill(pid, 0)
|
|
840
|
-
return true
|
|
841
|
-
} catch (error) {
|
|
842
|
-
return Boolean(error && typeof error === "object" && "code" in error && error.code === "EPERM")
|
|
843
|
-
}
|
|
844
|
-
}
|
|
845
|
-
|
|
846
|
-
/** @returns {void} */
|
|
847
|
-
function clearOwnerMutationLease() {
|
|
848
|
-
if (ownerMutationLeaseTimer) clearTimeout(ownerMutationLeaseTimer)
|
|
849
|
-
ownerMutationLeaseTimer = undefined
|
|
850
|
-
ownerMutationClient = undefined
|
|
851
|
-
ownerMutationId = undefined
|
|
852
|
-
ownerMutationOperation = undefined
|
|
853
|
-
ownerMutationWaitingFor = undefined
|
|
854
|
-
ownerMutationPid = undefined
|
|
855
|
-
ownerMutationStartedAt = undefined
|
|
856
|
-
ownerMutationLeaseMs = OWNER_MUTATION_LEASE_MS
|
|
857
|
-
}
|
|
858
|
-
|
|
859
|
-
/**
|
|
860
|
-
* Arms the lease bound for one mutation.
|
|
861
|
-
* @param {string} mutationId - Lease id the timer belongs to.
|
|
862
|
-
* @returns {void}
|
|
863
|
-
*/
|
|
864
|
-
function armOwnerMutationLease(mutationId) {
|
|
865
|
-
if (ownerMutationLeaseTimer) clearTimeout(ownerMutationLeaseTimer)
|
|
866
|
-
ownerMutationLeaseTimer = setTimeout(() => expireOwnerMutationLease(mutationId), ownerMutationLeaseMs)
|
|
867
|
-
ownerMutationLeaseTimer.unref()
|
|
868
|
-
}
|
|
869
|
-
|
|
870
|
-
/**
|
|
871
|
-
* A mutation that outruns its lease is reported and released instead of holding the fence
|
|
872
|
-
* forever, so a stuck mutation cannot lock out every later deploy. The holder learns the
|
|
873
|
-
* lease is gone when it tries to end it.
|
|
874
|
-
* @param {string} mutationId - Lease id the armed timer belongs to.
|
|
875
|
-
* @returns {void}
|
|
876
|
-
*/
|
|
877
|
-
function expireOwnerMutationLease(mutationId) {
|
|
878
|
-
ownerMutationLeaseTimer = undefined
|
|
879
|
-
if (!ownerMutationClient || ownerMutationId !== mutationId) return
|
|
880
|
-
const operation = ownerMutationOperation || "unnamed operation"
|
|
881
|
-
const leaseMs = ownerMutationLeaseMs
|
|
882
|
-
const elapsedMs = Math.max(0, Date.now() - (ownerMutationStartedAt ?? 0))
|
|
883
|
-
const diagnostic = {
|
|
884
|
-
code: "OWNER_MUTATION_LEASE_EXPIRED",
|
|
885
|
-
elapsedMs,
|
|
886
|
-
event: "owner-mutation-expired",
|
|
887
|
-
leaseMs,
|
|
888
|
-
mutationId,
|
|
889
|
-
operation,
|
|
890
|
-
pid: ownerMutationPid ?? ownerClientPid,
|
|
891
|
-
waitingFor: ownerMutationWaitingFor || "the operation to report completion"
|
|
892
|
-
}
|
|
893
|
-
|
|
894
|
-
process.stderr.write(`${JSON.stringify({at: new Date().toISOString(), ...diagnostic})}\n`)
|
|
895
|
-
if (!ownerMutationClient.destroyed) ownerMutationClient.write(`${JSON.stringify(diagnostic)}\n`)
|
|
896
|
-
broadcast(diagnostic)
|
|
897
|
-
ownerMutationExpired = {elapsedMs, leaseMs, mutationId, operation}
|
|
898
|
-
clearOwnerMutationLease()
|
|
899
|
-
}
|
|
900
|
-
|
|
901
708
|
/**
|
|
902
709
|
* @param {net.Socket} socket - Requesting client.
|
|
903
710
|
* @param {string} command - Command name.
|
|
@@ -918,7 +725,10 @@ function releaseClosedOwner(socket) {
|
|
|
918
725
|
rememberDisconnectedOwnerSuccessors(socket)
|
|
919
726
|
ownerClient = undefined
|
|
920
727
|
ownerClientPid = undefined
|
|
921
|
-
if (ownerMutationClient === socket)
|
|
728
|
+
if (ownerMutationClient === socket) {
|
|
729
|
+
ownerMutationClient = undefined
|
|
730
|
+
ownerMutationId = undefined
|
|
731
|
+
}
|
|
922
732
|
if (replacementClient && replacementOwnerState) commitReplacement()
|
|
923
733
|
else {
|
|
924
734
|
if (incompleteRestartChild) {
|
package/src/process-memory.js
CHANGED
|
@@ -4,14 +4,8 @@ import fs from "node:fs"
|
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* @typedef {{command: string, pid: number, rssBytes: number | undefined}} ProcessGroupMember
|
|
7
|
-
* @typedef {{cpuMs: number | undefined, liveMemberCount: number, memberCount: number}} ProcessGroupSample
|
|
8
7
|
*/
|
|
9
8
|
|
|
10
|
-
// Linux reports /proc/<pid>/stat CPU time in clock ticks and keeps USER_HZ at 100 for
|
|
11
|
-
// userspace, so ticks * 10 / 1 is milliseconds. This is a monotonic per-process counter,
|
|
12
|
-
// not a rate: two reads of it measure whether a process did any work between them.
|
|
13
|
-
const CLOCK_TICKS_PER_MS = 10
|
|
14
|
-
|
|
15
9
|
/**
|
|
16
10
|
* Lists the members of a managed process group with each member's resident memory.
|
|
17
11
|
* Rollbridge spawns each process detached, so the spawned pid is the process-group
|
|
@@ -89,51 +83,6 @@ export function measureProcessGroupRssBytes(pgid) {
|
|
|
89
83
|
return measured.reduce((total, member) => total + (member.rssBytes ?? 0), 0)
|
|
90
84
|
}
|
|
91
85
|
|
|
92
|
-
/**
|
|
93
|
-
* Samples one managed process group for drain liveness: how much CPU its members have
|
|
94
|
-
* consumed in total, and how many of them can still run.
|
|
95
|
-
*
|
|
96
|
-
* CPU is the sum of every member's cumulative user+system time, defunct members included:
|
|
97
|
-
* a member that exits takes its consumed time out of the sum, so the difference between two
|
|
98
|
-
* samples is non-zero whenever the group changed at all — new work or a departing member.
|
|
99
|
-
* Only an exactly unchanged difference says the group did nothing between the two reads.
|
|
100
|
-
* @param {number} pgid - Process-group id (the detached spawn's pid).
|
|
101
|
-
* @param {string} [procPath] - Procfs root (overridable for deterministic tests).
|
|
102
|
-
* @returns {ProcessGroupSample | undefined} The sample, or undefined only when procfs cannot be
|
|
103
|
-
* read at all. A group with no members left is a sample of zero, not a missing measurement.
|
|
104
|
-
*/
|
|
105
|
-
export function sampleProcessGroup(pgid, procPath = "/proc") {
|
|
106
|
-
/** @type {string[]} */
|
|
107
|
-
let entries
|
|
108
|
-
|
|
109
|
-
try {
|
|
110
|
-
entries = fs.readdirSync(procPath)
|
|
111
|
-
} catch {
|
|
112
|
-
return undefined
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
let cpuMs = 0
|
|
116
|
-
let liveMemberCount = 0
|
|
117
|
-
let memberCount = 0
|
|
118
|
-
let measurable = true
|
|
119
|
-
|
|
120
|
-
for (const entry of entries) {
|
|
121
|
-
if (!/^\d+$/.test(entry)) continue
|
|
122
|
-
|
|
123
|
-
const stat = processStat(entry, procPath)
|
|
124
|
-
|
|
125
|
-
if (stat?.pgrp !== pgid) continue
|
|
126
|
-
|
|
127
|
-
memberCount += 1
|
|
128
|
-
if (stat.state !== "Z" && stat.state !== "X") liveMemberCount += 1
|
|
129
|
-
|
|
130
|
-
if (stat.cpuMs === undefined) measurable = false
|
|
131
|
-
else cpuMs += stat.cpuMs
|
|
132
|
-
}
|
|
133
|
-
|
|
134
|
-
return {cpuMs: measurable ? cpuMs : undefined, liveMemberCount, memberCount}
|
|
135
|
-
}
|
|
136
|
-
|
|
137
86
|
/**
|
|
138
87
|
* @param {string} pid - Process id.
|
|
139
88
|
* @returns {string} The process command name (`/proc/<pid>/comm`), or "" when unavailable.
|
|
@@ -157,7 +106,7 @@ function processGroupId(pid) {
|
|
|
157
106
|
/**
|
|
158
107
|
* @param {string} pid - Process id.
|
|
159
108
|
* @param {string} procPath - Procfs root.
|
|
160
|
-
* @returns {{
|
|
109
|
+
* @returns {{pgrp: number, state: string} | undefined} Parsed process state and group.
|
|
161
110
|
*/
|
|
162
111
|
function processStat(pid, procPath) {
|
|
163
112
|
let stat
|
|
@@ -177,16 +126,7 @@ function processStat(pid, procPath) {
|
|
|
177
126
|
const fields = stat.slice(commEnd + 2).split(" ")
|
|
178
127
|
const pgrp = Number(fields[2])
|
|
179
128
|
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
// utime and stime are fields 14 and 15 of stat, i.e. index 11 and 12 after the comm field.
|
|
183
|
-
const utimeTicks = Number(fields[11])
|
|
184
|
-
const stimeTicks = Number(fields[12])
|
|
185
|
-
const cpuMs = Number.isFinite(utimeTicks) && Number.isFinite(stimeTicks)
|
|
186
|
-
? (utimeTicks + stimeTicks) * CLOCK_TICKS_PER_MS
|
|
187
|
-
: undefined
|
|
188
|
-
|
|
189
|
-
return {cpuMs, pgrp, state: fields[0]}
|
|
129
|
+
return Number.isInteger(pgrp) ? {pgrp, state: fields[0]} : undefined
|
|
190
130
|
}
|
|
191
131
|
|
|
192
132
|
/**
|