rollbridge 0.1.73 → 0.1.74

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/README.md +63 -10
  2. package/TODO.md +13 -0
  3. package/changelog.d/20261025-guardian-re-register-dead-generation.md +13 -0
  4. package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +1 -0
  5. package/changelog.d/20261104-compose-service-units.md +21 -0
  6. package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +20 -0
  7. package/changelog.d/20261106-release-dependency-boot-budgets.md +32 -0
  8. package/changelog.d/20261109-path-filtered-builds.md +11 -0
  9. package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +26 -0
  10. package/changelog.d/20261111-contention-sensitive-daemon-specs.md +27 -0
  11. package/changelog.d/20261113-bounded-nonblocking-retirement.md +5 -0
  12. package/changelog.d/20261115-bound-draining-release-groups.md +62 -0
  13. package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +27 -0
  14. package/changelog.d/20261115-wedged-generation-escalation.md +44 -0
  15. package/changelog.d/20261116-bounded-explicit-stop-retirement.md +30 -0
  16. package/changelog.d/20261116-config-json-canonical-shape.md +11 -0
  17. package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +29 -0
  18. package/changelog.d/20261116-guardian-version-handoff.md +37 -0
  19. package/changelog.d/20261116-reclaim-dead-owner-mutation.md +7 -0
  20. package/changelog.d/20261117-activate-before-incumbent-retirement.md +1 -0
  21. package/changelog.d/20261117-activation-path-drain-deadline.md +35 -0
  22. package/changelog.d/20261117-stop-companion-after-retired-dependency.md +7 -0
  23. package/changelog.d/20261118-incumbent-retirement-authority-guard.md +1 -0
  24. package/changelog.d/20261118-owner-retirement-no-guardian-bound.md +19 -0
  25. package/changelog.d/20261118-shutdown-retirement-bound-restored.md +31 -0
  26. package/docs/cli.md +128 -3
  27. package/docs/compose-services.md +222 -0
  28. package/docs/config.md +177 -6
  29. package/docs/troubleshooting.md +97 -6
  30. package/docs/velocious.md +1 -1
  31. package/docs/workers.md +11 -0
  32. package/examples/tensorbuzz.com.js +3 -2
  33. package/package.json +1 -1
  34. package/src/cli.js +68 -2
  35. package/src/compose-unit.js +725 -0
  36. package/src/config.js +346 -8
  37. package/src/daemon.js +411 -78
  38. package/src/doctor.js +74 -0
  39. package/src/drain-policy.js +252 -0
  40. package/src/guardian-client.js +68 -23
  41. package/src/health.js +99 -8
  42. package/src/managed-process.js +198 -45
  43. package/src/process-guardian.js +212 -37
  44. package/src/process-memory.js +58 -23
  45. package/src/process-order.js +26 -0
  46. package/src/release-group.js +438 -45
  47. package/tensorbuzz.yml +20 -1
  48. package/test/completion.test.js +5 -2
  49. package/test/compose-config.test.js +160 -0
  50. package/test/compose-docker.test.js +297 -0
  51. package/test/compose-unit.test.js +415 -0
  52. package/test/config-validation.test.js +135 -5
  53. package/test/drain-policy.test.js +328 -0
  54. package/test/fixtures/deferred-boot-app.js +106 -0
  55. package/test/fixtures/gated-app.js +50 -0
  56. package/test/guardian-client.test.js +119 -5
  57. package/test/health.test.js +74 -1
  58. package/test/managed-process.test.js +260 -24
  59. package/test/owner-recovery.test.js +125 -21
  60. package/test/owner-replacement.test.js +175 -14
  61. package/test/process-memory.test.js +47 -1
  62. package/test/release-group.test.js +408 -2
  63. package/test/rollbridge.test.js +774 -72
  64. package/test/shutdown-completion.test.js +119 -0
  65. package/test/shutdown-retirement-bound.test.js +162 -0
  66. package/test/support/process.js +3 -1
  67. package/test/tensorbuzz-config.test.js +228 -0
@@ -4,14 +4,21 @@ import fsSync from "node:fs"
4
4
  import fs from "node:fs/promises"
5
5
  import net from "node:net"
6
6
  import crypto from "node:crypto"
7
+ import {createRequire} from "node:module"
7
8
  import {spawn} from "node:child_process"
8
9
  import {isDeepStrictEqual} from "node:util"
9
10
  import path from "node:path"
10
11
  import GuardianClient from "./guardian-client.js"
11
- import ManagedProcess from "./managed-process.js"
12
+ import ManagedProcess, {childRecordIsGone} from "./managed-process.js"
12
13
  import {writeState} from "./state-store.js"
13
14
 
14
15
  const OWNER_RESTART_RETRY_MS = 1000
16
+ const commonJsRequire = createRequire(import.meta.url)
17
+ const GUARDIAN_VERSION = /** @type {{version: string}} */ (commonJsRequire("../package.json")).version
18
+ // An owner mutation is deploy/stop/restart/rollback/recovery work. A retained
19
+ // generation may legitimately drain for hours, but a mutation that holds the
20
+ // fence that long is stuck, so the lease is bounded and released on expiry.
21
+ const OWNER_MUTATION_LEASE_MS = 900000
15
22
 
16
23
  /** @typedef {import("node:child_process").ChildProcess["signalCode"]} ChildExitSignal */
17
24
 
@@ -24,7 +31,7 @@ const OWNER_RESTART_RETRY_MS = 1000
24
31
  * @property {number} id - Request id.
25
32
  * @property {string} [key] - Stable process key.
26
33
  * @property {boolean} [localSource] - Whether the retired daemon physically owns the published source.
27
- * @property {{timeoutMs?: number}} [options] - Stop options.
34
+ * @property {{retirementTimeoutMs?: number, timeoutMs?: number}} [options] - Stop options.
28
35
  * @property {string} [previousProvenance] - Expected current provenance.
29
36
  * @property {string} [provenance] - New or registered provenance.
30
37
  * @property {string} [releaseId] - Retained release identity.
@@ -32,11 +39,10 @@ const OWNER_RESTART_RETRY_MS = 1000
32
39
  * @property {import("./json.js").JsonValue} [ownerState] - Private owner transfer state.
33
40
  * @property {string} [replacementId] - Prepared replacement transaction id.
34
41
  * @property {string} [mutationId] - Owner mutation lease id.
42
+ * @property {number} [mutationTimeoutMs] - Bound for the owner mutation lease.
35
43
  * @property {string} [operation] - Owner mutation diagnostic name.
44
+ * @property {string} [waitingFor] - What the held owner mutation is blocked on.
36
45
  * @property {number} [ownerPid] - Exact claimant process PID.
37
- * @property {string} [configDigest] - Exact committed owner configuration digest.
38
- * @property {number} [pid] - Exact live process PID to re-adopt.
39
- * @property {string} [processStartTime] - Exact `/proc` process start-time counter.
40
46
  * @property {import("./json.js").JsonValue} [recoverySnapshot] - Public bridge recovery snapshot.
41
47
  * @property {string} [statePath] - Exact public recovery state path.
42
48
  * @property {import("./json.js").JsonValue} [authority] - Expected current authority.
@@ -106,6 +112,19 @@ const legacyKeys = legacyGuardian ? committedOwnerProcessKeys(ownerState) : new
106
112
  let ownerMutationClient
107
113
  /** @type {string | undefined} */
108
114
  let ownerMutationId
115
+ /** @type {string | undefined} */
116
+ let ownerMutationOperation
117
+ /** @type {string | undefined} */
118
+ let ownerMutationWaitingFor
119
+ /** @type {number | undefined} */
120
+ let ownerMutationPid
121
+ /** @type {number | undefined} */
122
+ let ownerMutationStartedAt
123
+ let ownerMutationLeaseMs = OWNER_MUTATION_LEASE_MS
124
+ /** @type {ReturnType<typeof setTimeout> | undefined} */
125
+ let ownerMutationLeaseTimer
126
+ /** @type {{elapsedMs: number, leaseMs: number, mutationId: string, operation: string} | undefined} */
127
+ let ownerMutationExpired
109
128
  /** @type {net.Socket | undefined} */
110
129
  let retiringClient
111
130
  /** @type {string | undefined} */
@@ -157,10 +176,7 @@ const server = net.createServer((socket) => {
157
176
  clearTimeout(waiter.timer)
158
177
  waiter.reject(new Error("Owner claimant disconnected"))
159
178
  }
160
- if (ownerMutationClient === socket && ownerClient !== socket) {
161
- ownerMutationClient = undefined
162
- ownerMutationId = undefined
163
- }
179
+ if (ownerMutationClient === socket) clearOwnerMutationLease()
164
180
  if (controlLessRetirement) {
165
181
  const preCommitHandoff = replacementClient && ownerClient === socket && retiringReplacementId === replacementId
166
182
 
@@ -262,13 +278,14 @@ async function handleLine(socket, line) {
262
278
  async function execute(request, socket) {
263
279
  if (shuttingDown) throw new Error("Process guardian is shutting down")
264
280
 
265
- if (request.command === "capabilities") return {daemonRecovery: 1, generationReactivation: 1}
281
+ if (request.command === "capabilities") return {daemonRecovery: 1, generationReactivation: 1, guardianVersion: GUARDIAN_VERSION}
266
282
 
267
283
  if (request.command === "owner-replacement-capabilities") {
268
284
  return {commands: ["commit-retired-owner-replacement"], protocol: "owner-replacement", version: 1}
269
285
  }
270
286
 
271
287
  if (request.command === "claim-owner") {
288
+ reclaimDeadOwnerMutation()
272
289
  if (retirementFailed) throw new Error("Guardian owner replacement is fenced after incomplete retired-listener state transfer")
273
290
  if (ownerState !== undefined && !isDeepStrictEqual(request.authority, ownerAuthority(ownerState))) {
274
291
  const waitsForPreparedAuthority = Boolean(ownerClient && replacementOwnerState && isDeepStrictEqual(request.authority, replacementAuthority))
@@ -304,8 +321,7 @@ async function execute(request, socket) {
304
321
  rememberDisconnectedOwnerSuccessors(socket)
305
322
  ownerClient = undefined
306
323
  ownerClientPid = undefined
307
- ownerMutationClient = undefined
308
- ownerMutationId = undefined
324
+ clearOwnerMutationLease()
309
325
  ownerRevision += 1
310
326
  grantNextOwner()
311
327
  return {retired: true}
@@ -397,29 +413,54 @@ async function execute(request, socket) {
397
413
  return {committedReplacementId: committedReplacementId ?? null, ownerClaimed: Boolean(ownerClient), retirementFailed, retirementPending: Boolean(retiringClient), retirementReady: retiringListenerReady}
398
414
  }
399
415
 
416
+ if (request.command === "guardian-status") return guardianStatus()
417
+
400
418
  if (request.command === "begin-owner-mutation") {
401
419
  requireOwner(socket, request.command)
420
+ reclaimDeadOwnerMutation()
402
421
  if (replacementClient) throw new Error(`Owner mutation ${request.operation || "operation"} is fenced while an owner replacement is prepared`)
403
- if (ownerMutationClient) throw new Error("Another owner mutation is already in progress")
422
+ const leaseMs = request.mutationTimeoutMs === undefined ? OWNER_MUTATION_LEASE_MS : request.mutationTimeoutMs
423
+
424
+ if (!Number.isInteger(leaseMs) || leaseMs <= 0) throw new Error("Owner mutation lease bound must be a positive number of milliseconds")
425
+ if (ownerMutationClient) throw new Error(`Another owner mutation is already in progress: ${describeOwnerMutationLease()}; refused ${request.operation || "operation"}`)
404
426
  ownerMutationClient = socket
405
427
  ownerMutationId = crypto.randomUUID()
406
- return {mutationId: ownerMutationId}
428
+ ownerMutationOperation = request.operation
429
+ ownerMutationWaitingFor = request.waitingFor
430
+ ownerMutationPid = request.ownerPid ?? ownerClientPid
431
+ ownerMutationStartedAt = Date.now()
432
+ ownerMutationLeaseMs = leaseMs
433
+ armOwnerMutationLease(ownerMutationId)
434
+ return {leaseMs, mutationId: ownerMutationId}
407
435
  }
408
436
 
409
437
  if (request.command === "end-owner-mutation") {
410
438
  requireOwner(socket, request.command)
439
+ if (ownerMutationExpired && request.mutationId === ownerMutationExpired.mutationId) {
440
+ const expired = ownerMutationExpired
441
+
442
+ ownerMutationExpired = undefined
443
+ throw new Error(`Owner mutation ${expired.operation} exceeded its ${expired.leaseMs} ms lease and was released after ${expired.elapsedMs} ms; the mutation failed and its lease is gone`)
444
+ }
411
445
  if (ownerMutationClient !== socket || request.mutationId !== ownerMutationId) throw new Error("Owner mutation lease does not match the active mutation")
412
- ownerMutationClient = undefined
413
- ownerMutationId = undefined
446
+ clearOwnerMutationLease()
414
447
  return {ended: true}
415
448
  }
416
449
 
450
+ if (request.command === "report-owner-mutation-wait") {
451
+ requireOwner(socket, request.command)
452
+ if (ownerMutationClient !== socket || request.mutationId !== ownerMutationId) throw new Error("Owner mutation lease does not match the active mutation")
453
+ if (typeof request.waitingFor !== "string" || !request.waitingFor.trim()) throw new Error("Owner mutation wait report requires a waitingFor description")
454
+ ownerMutationWaitingFor = request.waitingFor.trim()
455
+ return {reported: true}
456
+ }
457
+
417
458
  if (request.command === "prepare-owner-replacement") {
418
459
  if (retirementFailed) throw new Error("Guardian owner replacement is fenced after incomplete retired-listener state transfer")
419
460
  if (retiringClient) throw new Error("Guardian owner replacement cannot prepare while listener retirement is pending")
420
461
  if (socket === ownerClient) throw new Error("Committed owner cannot prepare itself as a replacement")
421
462
  if (replacementClient && replacementClient !== socket) throw new Error("Another owner replacement candidate is already prepared")
422
- if (ownerMutationClient) throw new Error("Owner replacement cannot prepare while an owner mutation is in progress")
463
+ if (ownerMutationClient) throw new Error(`Owner replacement cannot prepare while an owner mutation is in progress: ${describeOwnerMutationLease()}`)
423
464
  if (!ownerState) throw new Error("Committed owner has not published transferable state")
424
465
  const currentAuthority = ownerAuthority(ownerState)
425
466
  const resumesCommittedAuthority = !ownerClient && isDeepStrictEqual(currentAuthority, request.nextAuthority)
@@ -448,7 +489,7 @@ async function execute(request, socket) {
448
489
 
449
490
  if (request.command === "stage-owner-replacement") {
450
491
  requireReplacement(socket, request)
451
- if (replacementRevision !== ownerRevision) throw new Error("Guardian owner state changed after prepare; abort the stale candidate before preparing again")
492
+ if (replacementRevision !== ownerRevision) throw new Error(`Guardian owner state changed after prepare (prepared revision ${replacementRevision}, current revision ${ownerRevision}); abort the stale candidate before preparing again`)
452
493
  if (request.ownerState === undefined || !isDeepStrictEqual(ownerAuthority(request.ownerState), replacementAuthority)) {
453
494
  throw new Error("Prepared replacement state does not match the requested authority")
454
495
  }
@@ -579,7 +620,7 @@ async function execute(request, socket) {
579
620
  requireOwner(socket, request.command)
580
621
  shuttingDown = true
581
622
  try {
582
- const results = await Promise.allSettled([...processes.values()].map((entry) => entry.process.stop()))
623
+ const results = await Promise.allSettled([...processes.values()].map((entry) => entry.process.stop(request.options)))
583
624
  const errors = results.filter((result) => result.status === "rejected").map((result) => result.reason)
584
625
 
585
626
  if (errors.length > 0) throw new AggregateError(errors, `Guardian failed to stop ${errors.length} owned process${errors.length === 1 ? "" : "es"}: ${errors.map((error) => errorMessage(error instanceof Error ? error : String(error))).join("; ")}`)
@@ -616,8 +657,17 @@ async function execute(request, socket) {
616
657
  const existing = processes.get(request.key)
617
658
 
618
659
  if (existing) {
619
- if (existing.provenance !== request.provenance) throw new Error(`Guardian provenance mismatch for ${request.key}`)
620
- return existing.process.status()
660
+ if (existing.provenance === request.provenance) return existing.process.status()
661
+ // A provenance mismatch only fences a process that still exists. When the registered
662
+ // child is conclusively gone (stopped or failed with no live pid), the old record is
663
+ // residue of a dead generation, and demanding its provenance wedges every later deploy
664
+ // against a process that no longer exists (TensorBuzz production, STC-695). The owner
665
+ // may re-register the key under the new definition; live processes stay fail-closed.
666
+ // A group that outlived its tracked process is not gone: it still holds the key's work,
667
+ // so re-registering it would start a second copy of a generation that is still running.
668
+ if (!childRecordIsGone(existing.process.status())) throw new Error(`Guardian provenance mismatch for ${request.key}`)
669
+ requireOwner(socket, request.command)
670
+ processes.delete(request.key)
621
671
  }
622
672
 
623
673
  const recoversLegacyProcess = Boolean(legacyGuardian && legacyKeys.has(request.key))
@@ -660,18 +710,6 @@ async function execute(request, socket) {
660
710
  if (request.command === "start") {
661
711
  record.desired = true
662
712
  await record.process.start(request.reason, request.lifecycleRole)
663
- } else if (request.command === "adopt") {
664
- requireOwner(socket, request.command)
665
- const requestedPid = request.pid
666
- const requestedStartTime = request.processStartTime
667
- const committedConfigDigest = ownerState && typeof ownerState === "object" && !Array.isArray(ownerState)
668
- ? /** @type {{authority?: {configDigest?: string}}} */ (ownerState).authority?.configDigest
669
- : undefined
670
- if (!request.configDigest || !committedConfigDigest || !isDeepStrictEqual(request.configDigest, committedConfigDigest)) throw new Error("Guardian process adoption authority does not match the committed owner")
671
- if (!Number.isInteger(requestedPid) || !requestedStartTime) throw new Error("Guardian process adoption requires an exact PID and process start time")
672
- if (typeof record.process.adopt !== "function") throw new Error("Guardian process adoption is not supported by this process implementation")
673
- record.desired = true
674
- await record.process.adopt({pid: /** @type {number} */ (requestedPid), processStartTime: requestedStartTime}, request.configDigest)
675
713
  } else if (request.command === "activate") {
676
714
  await record.process.activateStrict()
677
715
  } else if (request.command === "reactivate" || request.command === "reactivate-with-command") {
@@ -720,6 +758,146 @@ async function execute(request, socket) {
720
758
  return status
721
759
  }
722
760
 
761
+ /**
762
+ * Names the mutation holding the owner-mutation fence: which mutation, which process,
763
+ * since when, and what it is blocked on. Refusals must be diagnosable without a live
764
+ * inspection of the daemon.
765
+ * @returns {string} Holder description used in refusal messages.
766
+ */
767
+ function describeOwnerMutationLease() {
768
+ const startedAt = ownerMutationStartedAt ?? 0
769
+ const elapsedSeconds = Math.max(0, Math.round((Date.now() - startedAt) / 1000))
770
+ const holderPid = ownerMutationPid ?? ownerClientPid
771
+
772
+ return [
773
+ `owner mutation "${ownerMutationOperation || "unnamed operation"}"`,
774
+ `mutation ${ownerMutationId || "unknown"}`,
775
+ `held by process ${holderPid ?? "unknown"}`,
776
+ `since ${new Date(startedAt).toISOString()}`,
777
+ `${elapsedSeconds}s elapsed`,
778
+ `waiting for ${ownerMutationWaitingFor || "the operation to report completion"}`
779
+ ].join("; ")
780
+ }
781
+
782
+ /**
783
+ * Reports the currently held owner-mutation lease.
784
+ * @returns {{elapsedMs: number, leaseMs: number, mutationId: string, operation: string, pid: number | null, startedAt: string, waitingFor: string} | null} Active lease, if any.
785
+ */
786
+ function activeOwnerMutationStatus() {
787
+ if (!ownerMutationClient || !ownerMutationId) return null
788
+
789
+ const startedAt = ownerMutationStartedAt ?? Date.now()
790
+
791
+ return {
792
+ elapsedMs: Math.max(0, Date.now() - startedAt),
793
+ leaseMs: ownerMutationLeaseMs,
794
+ mutationId: ownerMutationId,
795
+ operation: ownerMutationOperation || "unnamed operation",
796
+ pid: ownerMutationPid ?? ownerClientPid ?? null,
797
+ startedAt: new Date(startedAt).toISOString(),
798
+ waitingFor: ownerMutationWaitingFor || "the operation to report completion"
799
+ }
800
+ }
801
+
802
+ /**
803
+ * Reports guardian identity and owner-mutation diagnostics.
804
+ * @returns {{guardianPid: number, ownerMutation: ReturnType<typeof activeOwnerMutationStatus>, version: string}} Guardian diagnostics.
805
+ */
806
+ function guardianStatus() {
807
+ return {guardianPid: process.pid, ownerMutation: activeOwnerMutationStatus(), version: GUARDIAN_VERSION}
808
+ }
809
+
810
+ /** @returns {boolean} Whether the recorded owner-mutation holder is no longer alive. */
811
+ function reclaimDeadOwnerMutation() {
812
+ if (!ownerMutationClient || !ownerMutationPid || isProcessAlive(ownerMutationPid)) return false
813
+
814
+ const mutation = activeOwnerMutationStatus()
815
+ const deadOwnerSocket = ownerClient === ownerMutationClient && ownerClientPid === ownerMutationPid ? ownerClient : undefined
816
+ const diagnostic = {
817
+ code: "OWNER_MUTATION_RECLAIMED",
818
+ event: "owner-mutation-reclaimed",
819
+ mutationId: ownerMutationId,
820
+ operation: mutation?.operation || "unnamed operation",
821
+ pid: ownerMutationPid,
822
+ reason: "owner mutation holder process is no longer alive"
823
+ }
824
+
825
+ process.stderr.write(`${JSON.stringify({at: new Date().toISOString(), ...diagnostic})}\n`)
826
+ broadcast(diagnostic)
827
+ clearOwnerMutationLease()
828
+ if (deadOwnerSocket) releaseClosedOwner(deadOwnerSocket)
829
+ return true
830
+ }
831
+
832
+ /**
833
+ * Checks whether a process still exists without signaling it.
834
+ * @param {number} pid - Process id.
835
+ * @returns {boolean} Whether the process exists.
836
+ */
837
+ function isProcessAlive(pid) {
838
+ try {
839
+ process.kill(pid, 0)
840
+ return true
841
+ } catch (error) {
842
+ return Boolean(error && typeof error === "object" && "code" in error && error.code === "EPERM")
843
+ }
844
+ }
845
+
846
+ /** @returns {void} */
847
+ function clearOwnerMutationLease() {
848
+ if (ownerMutationLeaseTimer) clearTimeout(ownerMutationLeaseTimer)
849
+ ownerMutationLeaseTimer = undefined
850
+ ownerMutationClient = undefined
851
+ ownerMutationId = undefined
852
+ ownerMutationOperation = undefined
853
+ ownerMutationWaitingFor = undefined
854
+ ownerMutationPid = undefined
855
+ ownerMutationStartedAt = undefined
856
+ ownerMutationLeaseMs = OWNER_MUTATION_LEASE_MS
857
+ }
858
+
859
+ /**
860
+ * Arms the lease bound for one mutation.
861
+ * @param {string} mutationId - Lease id the timer belongs to.
862
+ * @returns {void}
863
+ */
864
+ function armOwnerMutationLease(mutationId) {
865
+ if (ownerMutationLeaseTimer) clearTimeout(ownerMutationLeaseTimer)
866
+ ownerMutationLeaseTimer = setTimeout(() => expireOwnerMutationLease(mutationId), ownerMutationLeaseMs)
867
+ ownerMutationLeaseTimer.unref()
868
+ }
869
+
870
+ /**
871
+ * A mutation that outruns its lease is reported and released instead of holding the fence
872
+ * forever, so a stuck mutation cannot lock out every later deploy. The holder learns the
873
+ * lease is gone when it tries to end it.
874
+ * @param {string} mutationId - Lease id the armed timer belongs to.
875
+ * @returns {void}
876
+ */
877
+ function expireOwnerMutationLease(mutationId) {
878
+ ownerMutationLeaseTimer = undefined
879
+ if (!ownerMutationClient || ownerMutationId !== mutationId) return
880
+ const operation = ownerMutationOperation || "unnamed operation"
881
+ const leaseMs = ownerMutationLeaseMs
882
+ const elapsedMs = Math.max(0, Date.now() - (ownerMutationStartedAt ?? 0))
883
+ const diagnostic = {
884
+ code: "OWNER_MUTATION_LEASE_EXPIRED",
885
+ elapsedMs,
886
+ event: "owner-mutation-expired",
887
+ leaseMs,
888
+ mutationId,
889
+ operation,
890
+ pid: ownerMutationPid ?? ownerClientPid,
891
+ waitingFor: ownerMutationWaitingFor || "the operation to report completion"
892
+ }
893
+
894
+ process.stderr.write(`${JSON.stringify({at: new Date().toISOString(), ...diagnostic})}\n`)
895
+ if (!ownerMutationClient.destroyed) ownerMutationClient.write(`${JSON.stringify(diagnostic)}\n`)
896
+ broadcast(diagnostic)
897
+ ownerMutationExpired = {elapsedMs, leaseMs, mutationId, operation}
898
+ clearOwnerMutationLease()
899
+ }
900
+
723
901
  /**
724
902
  * @param {net.Socket} socket - Requesting client.
725
903
  * @param {string} command - Command name.
@@ -740,10 +918,7 @@ function releaseClosedOwner(socket) {
740
918
  rememberDisconnectedOwnerSuccessors(socket)
741
919
  ownerClient = undefined
742
920
  ownerClientPid = undefined
743
- if (ownerMutationClient === socket) {
744
- ownerMutationClient = undefined
745
- ownerMutationId = undefined
746
- }
921
+ if (ownerMutationClient === socket) clearOwnerMutationLease()
747
922
  if (replacementClient && replacementOwnerState) commitReplacement()
748
923
  else {
749
924
  if (incompleteRestartChild) {
@@ -4,8 +4,14 @@ import fs from "node:fs"
4
4
 
5
5
  /**
6
6
  * @typedef {{command: string, pid: number, rssBytes: number | undefined}} ProcessGroupMember
7
+ * @typedef {{cpuMs: number | undefined, liveMemberCount: number, memberCount: number}} ProcessGroupSample
7
8
  */
8
9
 
10
+ // Linux reports /proc/<pid>/stat CPU time in clock ticks and keeps USER_HZ at 100 for
11
+ // userspace, so ticks * 10 / 1 is milliseconds. This is a monotonic per-process counter,
12
+ // not a rate: two reads of it measure whether a process did any work between them.
13
+ const CLOCK_TICKS_PER_MS = 10
14
+
9
15
  /**
10
16
  * Lists the members of a managed process group with each member's resident memory.
11
17
  * Rollbridge spawns each process detached, so the spawned pid is the process-group
@@ -71,41 +77,61 @@ export function processGroupHasLiveMembers(pgid, procPath = "/proc") {
71
77
  }
72
78
 
73
79
  /**
74
- * Returns Linux's monotonic process start-time counter for a PID.
75
- * @param {number} pid - Process id.
80
+ * Measures the total resident memory (RSS) of a managed process group.
81
+ * @param {number} pgid - Process-group id (the detached spawn's pid).
82
+ * @returns {number | undefined} Total resident memory in bytes, or undefined when unmeasurable.
83
+ */
84
+ export function measureProcessGroupRssBytes(pgid) {
85
+ const measured = processGroupMembers(pgid).filter((member) => member.rssBytes !== undefined)
86
+
87
+ if (measured.length === 0) return undefined
88
+
89
+ return measured.reduce((total, member) => total + (member.rssBytes ?? 0), 0)
90
+ }
91
+
92
+ /**
93
+ * Samples one managed process group for drain liveness: how much CPU its members have
94
+ * consumed in total, and how many of them can still run.
95
+ *
96
+ * CPU is the sum of every member's cumulative user+system time, defunct members included:
97
+ * a member that exits takes its consumed time out of the sum, so the difference between two
98
+ * samples is non-zero whenever the group changed at all — new work or a departing member.
99
+ * Only an exactly unchanged difference says the group did nothing between the two reads.
100
+ * @param {number} pgid - Process-group id (the detached spawn's pid).
76
101
  * @param {string} [procPath] - Procfs root (overridable for deterministic tests).
77
- * @returns {string | undefined} Kernel start-time ticks, or undefined when unavailable.
102
+ * @returns {ProcessGroupSample | undefined} The sample, or undefined only when procfs cannot be
103
+ * read at all. A group with no members left is a sample of zero, not a missing measurement.
78
104
  */
79
- export function processStartTime(pid, procPath = "/proc") {
80
- let stat
105
+ export function sampleProcessGroup(pgid, procPath = "/proc") {
106
+ /** @type {string[]} */
107
+ let entries
81
108
 
82
109
  try {
83
- stat = fs.readFileSync(`${procPath}/${pid}/stat`, "utf8")
110
+ entries = fs.readdirSync(procPath)
84
111
  } catch {
85
112
  return undefined
86
113
  }
87
114
 
88
- const commEnd = stat.lastIndexOf(")")
115
+ let cpuMs = 0
116
+ let liveMemberCount = 0
117
+ let memberCount = 0
118
+ let measurable = true
89
119
 
90
- if (commEnd < 0) return undefined
120
+ for (const entry of entries) {
121
+ if (!/^\d+$/.test(entry)) continue
91
122
 
92
- const fields = stat.slice(commEnd + 2).split(" ")
93
- const startTime = fields[19]
123
+ const stat = processStat(entry, procPath)
94
124
 
95
- return /^\d+$/.test(startTime || "") ? startTime : undefined
96
- }
125
+ if (stat?.pgrp !== pgid) continue
97
126
 
98
- /**
99
- * Measures the total resident memory (RSS) of a managed process group.
100
- * @param {number} pgid - Process-group id (the detached spawn's pid).
101
- * @returns {number | undefined} Total resident memory in bytes, or undefined when unmeasurable.
102
- */
103
- export function measureProcessGroupRssBytes(pgid) {
104
- const measured = processGroupMembers(pgid).filter((member) => member.rssBytes !== undefined)
127
+ memberCount += 1
128
+ if (stat.state !== "Z" && stat.state !== "X") liveMemberCount += 1
105
129
 
106
- if (measured.length === 0) return undefined
130
+ if (stat.cpuMs === undefined) measurable = false
131
+ else cpuMs += stat.cpuMs
132
+ }
107
133
 
108
- return measured.reduce((total, member) => total + (member.rssBytes ?? 0), 0)
134
+ return {cpuMs: measurable ? cpuMs : undefined, liveMemberCount, memberCount}
109
135
  }
110
136
 
111
137
  /**
@@ -131,7 +157,7 @@ function processGroupId(pid) {
131
157
  /**
132
158
  * @param {string} pid - Process id.
133
159
  * @param {string} procPath - Procfs root.
134
- * @returns {{pgrp: number, state: string} | undefined} Parsed process state and group.
160
+ * @returns {{cpuMs: number | undefined, pgrp: number, state: string} | undefined} Parsed process state, group, and cumulative CPU.
135
161
  */
136
162
  function processStat(pid, procPath) {
137
163
  let stat
@@ -151,7 +177,16 @@ function processStat(pid, procPath) {
151
177
  const fields = stat.slice(commEnd + 2).split(" ")
152
178
  const pgrp = Number(fields[2])
153
179
 
154
- return Number.isInteger(pgrp) ? {pgrp, state: fields[0]} : undefined
180
+ if (!Number.isInteger(pgrp)) return undefined
181
+
182
+ // utime and stime are fields 14 and 15 of stat, i.e. index 11 and 12 after the comm field.
183
+ const utimeTicks = Number(fields[11])
184
+ const stimeTicks = Number(fields[12])
185
+ const cpuMs = Number.isFinite(utimeTicks) && Number.isFinite(stimeTicks)
186
+ ? (utimeTicks + stimeTicks) * CLOCK_TICKS_PER_MS
187
+ : undefined
188
+
189
+ return {cpuMs, pgrp, state: fields[0]}
155
190
  }
156
191
 
157
192
  /**
@@ -0,0 +1,26 @@
1
+ // @ts-check
2
+
3
+ /**
4
+ * @typedef {import("./config.js").ProcessConfig} ProcessConfig
5
+ */
6
+
7
+ const POLICY_START_RANK = /** @type {Record<ProcessConfig["policy"], number>} */ ({companion: 1, proxied: 2, service: 0, singleton: 3})
8
+
9
+ /**
10
+ * Order in which Rollbridge starts processes, which is also the only order a `dependsOn`
11
+ * edge may point backwards in: daemon-wide persistent services are started before every
12
+ * release generation, then handoff services, companions, and the proxied process last.
13
+ * Config order is kept inside each group.
14
+ * @param {ProcessConfig[]} processes - Normalized processes.
15
+ * @param {{includePersistentServices?: boolean}} [options] - Whether to include the
16
+ * daemon-wide persistent services that a release group never starts itself.
17
+ * @returns {ProcessConfig[]} Ordered process configs.
18
+ */
19
+ export function processStartOrder(processes, {includePersistentServices = false} = {}) {
20
+ const persistentServices = includePersistentServices
21
+ ? processes.filter((processConfig) => processConfig.policy === "service" && processConfig.deployStrategy === "persistent")
22
+ : []
23
+ const generationProcesses = processes.filter((processConfig) => processConfig.policy !== "singleton" && (processConfig.policy !== "service" || processConfig.deployStrategy === "handoff"))
24
+
25
+ return [...persistentServices, ...generationProcesses.sort((left, right) => POLICY_START_RANK[left.policy] - POLICY_START_RANK[right.policy])]
26
+ }