rollbridge 0.1.72 → 0.1.74

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/README.md +63 -10
  2. package/TODO.md +13 -0
  3. package/changelog.d/20261025-guardian-re-register-dead-generation.md +13 -0
  4. package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +1 -0
  5. package/changelog.d/20261104-compose-service-units.md +21 -0
  6. package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +20 -0
  7. package/changelog.d/20261106-release-dependency-boot-budgets.md +32 -0
  8. package/changelog.d/20261109-path-filtered-builds.md +11 -0
  9. package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +26 -0
  10. package/changelog.d/20261111-contention-sensitive-daemon-specs.md +27 -0
  11. package/changelog.d/20261113-bounded-nonblocking-retirement.md +5 -0
  12. package/changelog.d/20261115-bound-draining-release-groups.md +62 -0
  13. package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +27 -0
  14. package/changelog.d/20261115-wedged-generation-escalation.md +44 -0
  15. package/changelog.d/20261116-bounded-explicit-stop-retirement.md +30 -0
  16. package/changelog.d/20261116-config-json-canonical-shape.md +11 -0
  17. package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +29 -0
  18. package/changelog.d/20261116-guardian-version-handoff.md +37 -0
  19. package/changelog.d/20261116-reclaim-dead-owner-mutation.md +7 -0
  20. package/changelog.d/20261117-activate-before-incumbent-retirement.md +1 -0
  21. package/changelog.d/20261117-activation-path-drain-deadline.md +35 -0
  22. package/changelog.d/20261117-stop-companion-after-retired-dependency.md +7 -0
  23. package/changelog.d/20261118-incumbent-retirement-authority-guard.md +1 -0
  24. package/changelog.d/20261118-owner-retirement-no-guardian-bound.md +19 -0
  25. package/changelog.d/20261118-shutdown-retirement-bound-restored.md +31 -0
  26. package/docs/cli.md +128 -3
  27. package/docs/compose-services.md +222 -0
  28. package/docs/config.md +177 -6
  29. package/docs/troubleshooting.md +97 -6
  30. package/docs/velocious.md +1 -1
  31. package/docs/workers.md +11 -0
  32. package/examples/tensorbuzz.com.js +3 -2
  33. package/package.json +1 -1
  34. package/src/cli.js +68 -2
  35. package/src/compose-unit.js +725 -0
  36. package/src/config.js +346 -8
  37. package/src/daemon.js +410 -73
  38. package/src/doctor.js +74 -0
  39. package/src/drain-policy.js +252 -0
  40. package/src/guardian-client.js +67 -8
  41. package/src/health.js +99 -8
  42. package/src/managed-process.js +197 -16
  43. package/src/process-guardian.js +212 -22
  44. package/src/process-memory.js +62 -2
  45. package/src/process-order.js +26 -0
  46. package/src/release-group.js +435 -37
  47. package/tensorbuzz.yml +20 -1
  48. package/test/completion.test.js +5 -2
  49. package/test/compose-config.test.js +160 -0
  50. package/test/compose-docker.test.js +297 -0
  51. package/test/compose-unit.test.js +415 -0
  52. package/test/config-validation.test.js +135 -5
  53. package/test/drain-policy.test.js +328 -0
  54. package/test/fixtures/deferred-boot-app.js +106 -0
  55. package/test/fixtures/gated-app.js +50 -0
  56. package/test/guardian-client.test.js +119 -5
  57. package/test/health.test.js +74 -1
  58. package/test/managed-process.test.js +260 -1
  59. package/test/owner-recovery.test.js +125 -21
  60. package/test/owner-replacement.test.js +175 -14
  61. package/test/process-memory.test.js +47 -1
  62. package/test/release-group.test.js +408 -2
  63. package/test/rollbridge.test.js +774 -72
  64. package/test/shutdown-completion.test.js +119 -0
  65. package/test/shutdown-retirement-bound.test.js +162 -0
  66. package/test/support/process.js +3 -1
  67. package/test/tensorbuzz-config.test.js +228 -0
@@ -4,14 +4,21 @@ import fsSync from "node:fs"
4
4
  import fs from "node:fs/promises"
5
5
  import net from "node:net"
6
6
  import crypto from "node:crypto"
7
+ import {createRequire} from "node:module"
7
8
  import {spawn} from "node:child_process"
8
9
  import {isDeepStrictEqual} from "node:util"
9
10
  import path from "node:path"
10
11
  import GuardianClient from "./guardian-client.js"
11
- import ManagedProcess from "./managed-process.js"
12
+ import ManagedProcess, {childRecordIsGone} from "./managed-process.js"
12
13
  import {writeState} from "./state-store.js"
13
14
 
14
15
  const OWNER_RESTART_RETRY_MS = 1000
16
+ const commonJsRequire = createRequire(import.meta.url)
17
+ const GUARDIAN_VERSION = /** @type {{version: string}} */ (commonJsRequire("../package.json")).version
18
+ // An owner mutation is deploy/stop/restart/rollback/recovery work. A retained
19
+ // generation may legitimately drain for hours, but a mutation that holds the
20
+ // fence that long is stuck, so the lease is bounded and released on expiry.
21
+ const OWNER_MUTATION_LEASE_MS = 900000
15
22
 
16
23
  /** @typedef {import("node:child_process").ChildProcess["signalCode"]} ChildExitSignal */
17
24
 
@@ -24,7 +31,7 @@ const OWNER_RESTART_RETRY_MS = 1000
24
31
  * @property {number} id - Request id.
25
32
  * @property {string} [key] - Stable process key.
26
33
  * @property {boolean} [localSource] - Whether the retired daemon physically owns the published source.
27
- * @property {{timeoutMs?: number}} [options] - Stop options.
34
+ * @property {{retirementTimeoutMs?: number, timeoutMs?: number}} [options] - Stop options.
28
35
  * @property {string} [previousProvenance] - Expected current provenance.
29
36
  * @property {string} [provenance] - New or registered provenance.
30
37
  * @property {string} [releaseId] - Retained release identity.
@@ -32,7 +39,9 @@ const OWNER_RESTART_RETRY_MS = 1000
32
39
  * @property {import("./json.js").JsonValue} [ownerState] - Private owner transfer state.
33
40
  * @property {string} [replacementId] - Prepared replacement transaction id.
34
41
  * @property {string} [mutationId] - Owner mutation lease id.
42
+ * @property {number} [mutationTimeoutMs] - Bound for the owner mutation lease.
35
43
  * @property {string} [operation] - Owner mutation diagnostic name.
44
+ * @property {string} [waitingFor] - What the held owner mutation is blocked on.
36
45
  * @property {number} [ownerPid] - Exact claimant process PID.
37
46
  * @property {import("./json.js").JsonValue} [recoverySnapshot] - Public bridge recovery snapshot.
38
47
  * @property {string} [statePath] - Exact public recovery state path.
@@ -103,6 +112,19 @@ const legacyKeys = legacyGuardian ? committedOwnerProcessKeys(ownerState) : new
103
112
  let ownerMutationClient
104
113
  /** @type {string | undefined} */
105
114
  let ownerMutationId
115
+ /** @type {string | undefined} */
116
+ let ownerMutationOperation
117
+ /** @type {string | undefined} */
118
+ let ownerMutationWaitingFor
119
+ /** @type {number | undefined} */
120
+ let ownerMutationPid
121
+ /** @type {number | undefined} */
122
+ let ownerMutationStartedAt
123
+ let ownerMutationLeaseMs = OWNER_MUTATION_LEASE_MS
124
+ /** @type {ReturnType<typeof setTimeout> | undefined} */
125
+ let ownerMutationLeaseTimer
126
+ /** @type {{elapsedMs: number, leaseMs: number, mutationId: string, operation: string} | undefined} */
127
+ let ownerMutationExpired
106
128
  /** @type {net.Socket | undefined} */
107
129
  let retiringClient
108
130
  /** @type {string | undefined} */
@@ -154,10 +176,7 @@ const server = net.createServer((socket) => {
154
176
  clearTimeout(waiter.timer)
155
177
  waiter.reject(new Error("Owner claimant disconnected"))
156
178
  }
157
- if (ownerMutationClient === socket && ownerClient !== socket) {
158
- ownerMutationClient = undefined
159
- ownerMutationId = undefined
160
- }
179
+ if (ownerMutationClient === socket) clearOwnerMutationLease()
161
180
  if (controlLessRetirement) {
162
181
  const preCommitHandoff = replacementClient && ownerClient === socket && retiringReplacementId === replacementId
163
182
 
@@ -259,13 +278,14 @@ async function handleLine(socket, line) {
259
278
  async function execute(request, socket) {
260
279
  if (shuttingDown) throw new Error("Process guardian is shutting down")
261
280
 
262
- if (request.command === "capabilities") return {daemonRecovery: 1, generationReactivation: 1}
281
+ if (request.command === "capabilities") return {daemonRecovery: 1, generationReactivation: 1, guardianVersion: GUARDIAN_VERSION}
263
282
 
264
283
  if (request.command === "owner-replacement-capabilities") {
265
284
  return {commands: ["commit-retired-owner-replacement"], protocol: "owner-replacement", version: 1}
266
285
  }
267
286
 
268
287
  if (request.command === "claim-owner") {
288
+ reclaimDeadOwnerMutation()
269
289
  if (retirementFailed) throw new Error("Guardian owner replacement is fenced after incomplete retired-listener state transfer")
270
290
  if (ownerState !== undefined && !isDeepStrictEqual(request.authority, ownerAuthority(ownerState))) {
271
291
  const waitsForPreparedAuthority = Boolean(ownerClient && replacementOwnerState && isDeepStrictEqual(request.authority, replacementAuthority))
@@ -301,8 +321,7 @@ async function execute(request, socket) {
301
321
  rememberDisconnectedOwnerSuccessors(socket)
302
322
  ownerClient = undefined
303
323
  ownerClientPid = undefined
304
- ownerMutationClient = undefined
305
- ownerMutationId = undefined
324
+ clearOwnerMutationLease()
306
325
  ownerRevision += 1
307
326
  grantNextOwner()
308
327
  return {retired: true}
@@ -394,29 +413,54 @@ async function execute(request, socket) {
394
413
  return {committedReplacementId: committedReplacementId ?? null, ownerClaimed: Boolean(ownerClient), retirementFailed, retirementPending: Boolean(retiringClient), retirementReady: retiringListenerReady}
395
414
  }
396
415
 
416
+ if (request.command === "guardian-status") return guardianStatus()
417
+
397
418
  if (request.command === "begin-owner-mutation") {
398
419
  requireOwner(socket, request.command)
420
+ reclaimDeadOwnerMutation()
399
421
  if (replacementClient) throw new Error(`Owner mutation ${request.operation || "operation"} is fenced while an owner replacement is prepared`)
400
- if (ownerMutationClient) throw new Error("Another owner mutation is already in progress")
422
+ const leaseMs = request.mutationTimeoutMs === undefined ? OWNER_MUTATION_LEASE_MS : request.mutationTimeoutMs
423
+
424
+ if (!Number.isInteger(leaseMs) || leaseMs <= 0) throw new Error("Owner mutation lease bound must be a positive number of milliseconds")
425
+ if (ownerMutationClient) throw new Error(`Another owner mutation is already in progress: ${describeOwnerMutationLease()}; refused ${request.operation || "operation"}`)
401
426
  ownerMutationClient = socket
402
427
  ownerMutationId = crypto.randomUUID()
403
- return {mutationId: ownerMutationId}
428
+ ownerMutationOperation = request.operation
429
+ ownerMutationWaitingFor = request.waitingFor
430
+ ownerMutationPid = request.ownerPid ?? ownerClientPid
431
+ ownerMutationStartedAt = Date.now()
432
+ ownerMutationLeaseMs = leaseMs
433
+ armOwnerMutationLease(ownerMutationId)
434
+ return {leaseMs, mutationId: ownerMutationId}
404
435
  }
405
436
 
406
437
  if (request.command === "end-owner-mutation") {
407
438
  requireOwner(socket, request.command)
439
+ if (ownerMutationExpired && request.mutationId === ownerMutationExpired.mutationId) {
440
+ const expired = ownerMutationExpired
441
+
442
+ ownerMutationExpired = undefined
443
+ throw new Error(`Owner mutation ${expired.operation} exceeded its ${expired.leaseMs} ms lease and was released after ${expired.elapsedMs} ms; the mutation failed and its lease is gone`)
444
+ }
408
445
  if (ownerMutationClient !== socket || request.mutationId !== ownerMutationId) throw new Error("Owner mutation lease does not match the active mutation")
409
- ownerMutationClient = undefined
410
- ownerMutationId = undefined
446
+ clearOwnerMutationLease()
411
447
  return {ended: true}
412
448
  }
413
449
 
450
+ if (request.command === "report-owner-mutation-wait") {
451
+ requireOwner(socket, request.command)
452
+ if (ownerMutationClient !== socket || request.mutationId !== ownerMutationId) throw new Error("Owner mutation lease does not match the active mutation")
453
+ if (typeof request.waitingFor !== "string" || !request.waitingFor.trim()) throw new Error("Owner mutation wait report requires a waitingFor description")
454
+ ownerMutationWaitingFor = request.waitingFor.trim()
455
+ return {reported: true}
456
+ }
457
+
414
458
  if (request.command === "prepare-owner-replacement") {
415
459
  if (retirementFailed) throw new Error("Guardian owner replacement is fenced after incomplete retired-listener state transfer")
416
460
  if (retiringClient) throw new Error("Guardian owner replacement cannot prepare while listener retirement is pending")
417
461
  if (socket === ownerClient) throw new Error("Committed owner cannot prepare itself as a replacement")
418
462
  if (replacementClient && replacementClient !== socket) throw new Error("Another owner replacement candidate is already prepared")
419
- if (ownerMutationClient) throw new Error("Owner replacement cannot prepare while an owner mutation is in progress")
463
+ if (ownerMutationClient) throw new Error(`Owner replacement cannot prepare while an owner mutation is in progress: ${describeOwnerMutationLease()}`)
420
464
  if (!ownerState) throw new Error("Committed owner has not published transferable state")
421
465
  const currentAuthority = ownerAuthority(ownerState)
422
466
  const resumesCommittedAuthority = !ownerClient && isDeepStrictEqual(currentAuthority, request.nextAuthority)
@@ -445,7 +489,7 @@ async function execute(request, socket) {
445
489
 
446
490
  if (request.command === "stage-owner-replacement") {
447
491
  requireReplacement(socket, request)
448
- if (replacementRevision !== ownerRevision) throw new Error("Guardian owner state changed after prepare; abort the stale candidate before preparing again")
492
+ if (replacementRevision !== ownerRevision) throw new Error(`Guardian owner state changed after prepare (prepared revision ${replacementRevision}, current revision ${ownerRevision}); abort the stale candidate before preparing again`)
449
493
  if (request.ownerState === undefined || !isDeepStrictEqual(ownerAuthority(request.ownerState), replacementAuthority)) {
450
494
  throw new Error("Prepared replacement state does not match the requested authority")
451
495
  }
@@ -576,7 +620,7 @@ async function execute(request, socket) {
576
620
  requireOwner(socket, request.command)
577
621
  shuttingDown = true
578
622
  try {
579
- const results = await Promise.allSettled([...processes.values()].map((entry) => entry.process.stop()))
623
+ const results = await Promise.allSettled([...processes.values()].map((entry) => entry.process.stop(request.options)))
580
624
  const errors = results.filter((result) => result.status === "rejected").map((result) => result.reason)
581
625
 
582
626
  if (errors.length > 0) throw new AggregateError(errors, `Guardian failed to stop ${errors.length} owned process${errors.length === 1 ? "" : "es"}: ${errors.map((error) => errorMessage(error instanceof Error ? error : String(error))).join("; ")}`)
@@ -613,8 +657,17 @@ async function execute(request, socket) {
613
657
  const existing = processes.get(request.key)
614
658
 
615
659
  if (existing) {
616
- if (existing.provenance !== request.provenance) throw new Error(`Guardian provenance mismatch for ${request.key}`)
617
- return existing.process.status()
660
+ if (existing.provenance === request.provenance) return existing.process.status()
661
+ // A provenance mismatch only fences a process that still exists. When the registered
662
+ // child is conclusively gone (stopped or failed with no live pid), the old record is
663
+ // residue of a dead generation, and demanding its provenance wedges every later deploy
664
+ // against a process that no longer exists (TensorBuzz production, STC-695). The owner
665
+ // may re-register the key under the new definition; live processes stay fail-closed.
666
+ // A group that outlived its tracked process is not gone: it still holds the key's work,
667
+ // so re-registering it would start a second copy of a generation that is still running.
668
+ if (!childRecordIsGone(existing.process.status())) throw new Error(`Guardian provenance mismatch for ${request.key}`)
669
+ requireOwner(socket, request.command)
670
+ processes.delete(request.key)
618
671
  }
619
672
 
620
673
  const recoversLegacyProcess = Boolean(legacyGuardian && legacyKeys.has(request.key))
@@ -705,6 +758,146 @@ async function execute(request, socket) {
705
758
  return status
706
759
  }
707
760
 
761
+ /**
762
+ * Names the mutation holding the owner-mutation fence: which mutation, which process,
763
+ * since when, and what it is blocked on. Refusals must be diagnosable without a live
764
+ * inspection of the daemon.
765
+ * @returns {string} Holder description used in refusal messages.
766
+ */
767
+ function describeOwnerMutationLease() {
768
+ const startedAt = ownerMutationStartedAt ?? 0
769
+ const elapsedSeconds = Math.max(0, Math.round((Date.now() - startedAt) / 1000))
770
+ const holderPid = ownerMutationPid ?? ownerClientPid
771
+
772
+ return [
773
+ `owner mutation "${ownerMutationOperation || "unnamed operation"}"`,
774
+ `mutation ${ownerMutationId || "unknown"}`,
775
+ `held by process ${holderPid ?? "unknown"}`,
776
+ `since ${new Date(startedAt).toISOString()}`,
777
+ `${elapsedSeconds}s elapsed`,
778
+ `waiting for ${ownerMutationWaitingFor || "the operation to report completion"}`
779
+ ].join("; ")
780
+ }
781
+
782
+ /**
783
+ * Reports the currently held owner-mutation lease.
784
+ * @returns {{elapsedMs: number, leaseMs: number, mutationId: string, operation: string, pid: number | null, startedAt: string, waitingFor: string} | null} Active lease, if any.
785
+ */
786
+ function activeOwnerMutationStatus() {
787
+ if (!ownerMutationClient || !ownerMutationId) return null
788
+
789
+ const startedAt = ownerMutationStartedAt ?? Date.now()
790
+
791
+ return {
792
+ elapsedMs: Math.max(0, Date.now() - startedAt),
793
+ leaseMs: ownerMutationLeaseMs,
794
+ mutationId: ownerMutationId,
795
+ operation: ownerMutationOperation || "unnamed operation",
796
+ pid: ownerMutationPid ?? ownerClientPid ?? null,
797
+ startedAt: new Date(startedAt).toISOString(),
798
+ waitingFor: ownerMutationWaitingFor || "the operation to report completion"
799
+ }
800
+ }
801
+
802
+ /**
803
+ * Reports guardian identity and owner-mutation diagnostics.
804
+ * @returns {{guardianPid: number, ownerMutation: ReturnType<typeof activeOwnerMutationStatus>, version: string}} Guardian diagnostics.
805
+ */
806
+ function guardianStatus() {
807
+ return {guardianPid: process.pid, ownerMutation: activeOwnerMutationStatus(), version: GUARDIAN_VERSION}
808
+ }
809
+
810
+ /** @returns {boolean} Whether the recorded owner-mutation holder is no longer alive. */
811
+ function reclaimDeadOwnerMutation() {
812
+ if (!ownerMutationClient || !ownerMutationPid || isProcessAlive(ownerMutationPid)) return false
813
+
814
+ const mutation = activeOwnerMutationStatus()
815
+ const deadOwnerSocket = ownerClient === ownerMutationClient && ownerClientPid === ownerMutationPid ? ownerClient : undefined
816
+ const diagnostic = {
817
+ code: "OWNER_MUTATION_RECLAIMED",
818
+ event: "owner-mutation-reclaimed",
819
+ mutationId: ownerMutationId,
820
+ operation: mutation?.operation || "unnamed operation",
821
+ pid: ownerMutationPid,
822
+ reason: "owner mutation holder process is no longer alive"
823
+ }
824
+
825
+ process.stderr.write(`${JSON.stringify({at: new Date().toISOString(), ...diagnostic})}\n`)
826
+ broadcast(diagnostic)
827
+ clearOwnerMutationLease()
828
+ if (deadOwnerSocket) releaseClosedOwner(deadOwnerSocket)
829
+ return true
830
+ }
831
+
832
+ /**
833
+ * Checks whether a process still exists without signaling it.
834
+ * @param {number} pid - Process id.
835
+ * @returns {boolean} Whether the process exists.
836
+ */
837
+ function isProcessAlive(pid) {
838
+ try {
839
+ process.kill(pid, 0)
840
+ return true
841
+ } catch (error) {
842
+ return Boolean(error && typeof error === "object" && "code" in error && error.code === "EPERM")
843
+ }
844
+ }
845
+
846
+ /** @returns {void} */
847
+ function clearOwnerMutationLease() {
848
+ if (ownerMutationLeaseTimer) clearTimeout(ownerMutationLeaseTimer)
849
+ ownerMutationLeaseTimer = undefined
850
+ ownerMutationClient = undefined
851
+ ownerMutationId = undefined
852
+ ownerMutationOperation = undefined
853
+ ownerMutationWaitingFor = undefined
854
+ ownerMutationPid = undefined
855
+ ownerMutationStartedAt = undefined
856
+ ownerMutationLeaseMs = OWNER_MUTATION_LEASE_MS
857
+ }
858
+
859
+ /**
860
+ * Arms the lease bound for one mutation.
861
+ * @param {string} mutationId - Lease id the timer belongs to.
862
+ * @returns {void}
863
+ */
864
+ function armOwnerMutationLease(mutationId) {
865
+ if (ownerMutationLeaseTimer) clearTimeout(ownerMutationLeaseTimer)
866
+ ownerMutationLeaseTimer = setTimeout(() => expireOwnerMutationLease(mutationId), ownerMutationLeaseMs)
867
+ ownerMutationLeaseTimer.unref()
868
+ }
869
+
870
+ /**
871
+ * A mutation that outruns its lease is reported and released instead of holding the fence
872
+ * forever, so a stuck mutation cannot lock out every later deploy. The holder learns the
873
+ * lease is gone when it tries to end it.
874
+ * @param {string} mutationId - Lease id the armed timer belongs to.
875
+ * @returns {void}
876
+ */
877
+ function expireOwnerMutationLease(mutationId) {
878
+ ownerMutationLeaseTimer = undefined
879
+ if (!ownerMutationClient || ownerMutationId !== mutationId) return
880
+ const operation = ownerMutationOperation || "unnamed operation"
881
+ const leaseMs = ownerMutationLeaseMs
882
+ const elapsedMs = Math.max(0, Date.now() - (ownerMutationStartedAt ?? 0))
883
+ const diagnostic = {
884
+ code: "OWNER_MUTATION_LEASE_EXPIRED",
885
+ elapsedMs,
886
+ event: "owner-mutation-expired",
887
+ leaseMs,
888
+ mutationId,
889
+ operation,
890
+ pid: ownerMutationPid ?? ownerClientPid,
891
+ waitingFor: ownerMutationWaitingFor || "the operation to report completion"
892
+ }
893
+
894
+ process.stderr.write(`${JSON.stringify({at: new Date().toISOString(), ...diagnostic})}\n`)
895
+ if (!ownerMutationClient.destroyed) ownerMutationClient.write(`${JSON.stringify(diagnostic)}\n`)
896
+ broadcast(diagnostic)
897
+ ownerMutationExpired = {elapsedMs, leaseMs, mutationId, operation}
898
+ clearOwnerMutationLease()
899
+ }
900
+
708
901
  /**
709
902
  * @param {net.Socket} socket - Requesting client.
710
903
  * @param {string} command - Command name.
@@ -725,10 +918,7 @@ function releaseClosedOwner(socket) {
725
918
  rememberDisconnectedOwnerSuccessors(socket)
726
919
  ownerClient = undefined
727
920
  ownerClientPid = undefined
728
- if (ownerMutationClient === socket) {
729
- ownerMutationClient = undefined
730
- ownerMutationId = undefined
731
- }
921
+ if (ownerMutationClient === socket) clearOwnerMutationLease()
732
922
  if (replacementClient && replacementOwnerState) commitReplacement()
733
923
  else {
734
924
  if (incompleteRestartChild) {
@@ -4,8 +4,14 @@ import fs from "node:fs"
4
4
 
5
5
  /**
6
6
  * @typedef {{command: string, pid: number, rssBytes: number | undefined}} ProcessGroupMember
7
+ * @typedef {{cpuMs: number | undefined, liveMemberCount: number, memberCount: number}} ProcessGroupSample
7
8
  */
8
9
 
10
+ // Linux reports /proc/<pid>/stat CPU time in clock ticks and keeps USER_HZ at 100 for
11
+ // userspace, so ticks * 10 / 1 is milliseconds. This is a monotonic per-process counter,
12
+ // not a rate: two reads of it measure whether a process did any work between them.
13
+ const CLOCK_TICKS_PER_MS = 10
14
+
9
15
  /**
10
16
  * Lists the members of a managed process group with each member's resident memory.
11
17
  * Rollbridge spawns each process detached, so the spawned pid is the process-group
@@ -83,6 +89,51 @@ export function measureProcessGroupRssBytes(pgid) {
83
89
  return measured.reduce((total, member) => total + (member.rssBytes ?? 0), 0)
84
90
  }
85
91
 
92
+ /**
93
+ * Samples one managed process group for drain liveness: how much CPU its members have
94
+ * consumed in total, and how many of them can still run.
95
+ *
96
+ * CPU is the sum of every member's cumulative user+system time, defunct members included:
97
+ * a member that exits takes its consumed time out of the sum, so the difference between two
98
+ * samples is non-zero whenever the group changed at all — new work or a departing member.
99
+ * Only an exactly unchanged difference says the group did nothing between the two reads.
100
+ * @param {number} pgid - Process-group id (the detached spawn's pid).
101
+ * @param {string} [procPath] - Procfs root (overridable for deterministic tests).
102
+ * @returns {ProcessGroupSample | undefined} The sample, or undefined only when procfs cannot be
103
+ * read at all. A group with no members left is a sample of zero, not a missing measurement.
104
+ */
105
+ export function sampleProcessGroup(pgid, procPath = "/proc") {
106
+ /** @type {string[]} */
107
+ let entries
108
+
109
+ try {
110
+ entries = fs.readdirSync(procPath)
111
+ } catch {
112
+ return undefined
113
+ }
114
+
115
+ let cpuMs = 0
116
+ let liveMemberCount = 0
117
+ let memberCount = 0
118
+ let measurable = true
119
+
120
+ for (const entry of entries) {
121
+ if (!/^\d+$/.test(entry)) continue
122
+
123
+ const stat = processStat(entry, procPath)
124
+
125
+ if (stat?.pgrp !== pgid) continue
126
+
127
+ memberCount += 1
128
+ if (stat.state !== "Z" && stat.state !== "X") liveMemberCount += 1
129
+
130
+ if (stat.cpuMs === undefined) measurable = false
131
+ else cpuMs += stat.cpuMs
132
+ }
133
+
134
+ return {cpuMs: measurable ? cpuMs : undefined, liveMemberCount, memberCount}
135
+ }
136
+
86
137
  /**
87
138
  * @param {string} pid - Process id.
88
139
  * @returns {string} The process command name (`/proc/<pid>/comm`), or "" when unavailable.
@@ -106,7 +157,7 @@ function processGroupId(pid) {
106
157
  /**
107
158
  * @param {string} pid - Process id.
108
159
  * @param {string} procPath - Procfs root.
109
- * @returns {{pgrp: number, state: string} | undefined} Parsed process state and group.
160
+ * @returns {{cpuMs: number | undefined, pgrp: number, state: string} | undefined} Parsed process state, group, and cumulative CPU.
110
161
  */
111
162
  function processStat(pid, procPath) {
112
163
  let stat
@@ -126,7 +177,16 @@ function processStat(pid, procPath) {
126
177
  const fields = stat.slice(commEnd + 2).split(" ")
127
178
  const pgrp = Number(fields[2])
128
179
 
129
- return Number.isInteger(pgrp) ? {pgrp, state: fields[0]} : undefined
180
+ if (!Number.isInteger(pgrp)) return undefined
181
+
182
+ // utime and stime are fields 14 and 15 of stat, i.e. index 11 and 12 after the comm field.
183
+ const utimeTicks = Number(fields[11])
184
+ const stimeTicks = Number(fields[12])
185
+ const cpuMs = Number.isFinite(utimeTicks) && Number.isFinite(stimeTicks)
186
+ ? (utimeTicks + stimeTicks) * CLOCK_TICKS_PER_MS
187
+ : undefined
188
+
189
+ return {cpuMs, pgrp, state: fields[0]}
130
190
  }
131
191
 
132
192
  /**
@@ -0,0 +1,26 @@
1
+ // @ts-check
2
+
3
+ /**
4
+ * @typedef {import("./config.js").ProcessConfig} ProcessConfig
5
+ */
6
+
7
+ const POLICY_START_RANK = /** @type {Record<ProcessConfig["policy"], number>} */ ({companion: 1, proxied: 2, service: 0, singleton: 3})
8
+
9
+ /**
10
+ * Order in which Rollbridge starts processes, which is also the only order a `dependsOn`
11
+ * edge may point backwards in: daemon-wide persistent services are started before every
12
+ * release generation, then handoff services, companions, and the proxied process last.
13
+ * Config order is kept inside each group.
14
+ * @param {ProcessConfig[]} processes - Normalized processes.
15
+ * @param {{includePersistentServices?: boolean}} [options] - Whether to include the
16
+ * daemon-wide persistent services that a release group never starts itself.
17
+ * @returns {ProcessConfig[]} Ordered process configs.
18
+ */
19
+ export function processStartOrder(processes, {includePersistentServices = false} = {}) {
20
+ const persistentServices = includePersistentServices
21
+ ? processes.filter((processConfig) => processConfig.policy === "service" && processConfig.deployStrategy === "persistent")
22
+ : []
23
+ const generationProcesses = processes.filter((processConfig) => processConfig.policy !== "singleton" && (processConfig.policy !== "service" || processConfig.deployStrategy === "handoff"))
24
+
25
+ return [...persistentServices, ...generationProcesses.sort((left, right) => POLICY_START_RANK[left.policy] - POLICY_START_RANK[right.policy])]
26
+ }