rollbridge 0.1.73 → 0.1.75

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/README.md +63 -10
  2. package/TODO.md +13 -0
  3. package/changelog.d/20261025-guardian-re-register-dead-generation.md +13 -0
  4. package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +1 -0
  5. package/changelog.d/20261104-compose-service-units.md +21 -0
  6. package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +20 -0
  7. package/changelog.d/20261106-release-dependency-boot-budgets.md +32 -0
  8. package/changelog.d/20261109-path-filtered-builds.md +11 -0
  9. package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +26 -0
  10. package/changelog.d/20261111-contention-sensitive-daemon-specs.md +27 -0
  11. package/changelog.d/20261113-bounded-nonblocking-retirement.md +5 -0
  12. package/changelog.d/20261115-bound-draining-release-groups.md +62 -0
  13. package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +27 -0
  14. package/changelog.d/20261115-wedged-generation-escalation.md +44 -0
  15. package/changelog.d/20261116-bounded-explicit-stop-retirement.md +30 -0
  16. package/changelog.d/20261116-config-json-canonical-shape.md +11 -0
  17. package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +29 -0
  18. package/changelog.d/20261116-guardian-version-handoff.md +37 -0
  19. package/changelog.d/20261116-reclaim-dead-owner-mutation.md +7 -0
  20. package/changelog.d/20261117-activate-before-incumbent-retirement.md +1 -0
  21. package/changelog.d/20261117-activation-path-drain-deadline.md +35 -0
  22. package/changelog.d/20261117-stop-companion-after-retired-dependency.md +7 -0
  23. package/changelog.d/20261118-bounded-control-client-close.md +4 -0
  24. package/changelog.d/20261118-incumbent-retirement-authority-guard.md +1 -0
  25. package/changelog.d/20261118-owner-retirement-no-guardian-bound.md +19 -0
  26. package/changelog.d/20261118-shutdown-retirement-bound-restored.md +31 -0
  27. package/docs/cli.md +128 -3
  28. package/docs/compose-services.md +222 -0
  29. package/docs/config.md +177 -6
  30. package/docs/troubleshooting.md +97 -6
  31. package/docs/velocious.md +1 -1
  32. package/docs/workers.md +11 -0
  33. package/examples/tensorbuzz.com.js +3 -2
  34. package/package.json +1 -1
  35. package/src/cli.js +69 -3
  36. package/src/compose-unit.js +725 -0
  37. package/src/config.js +346 -8
  38. package/src/control-client.js +32 -5
  39. package/src/daemon.js +448 -76
  40. package/src/doctor.js +74 -0
  41. package/src/drain-policy.js +252 -0
  42. package/src/guardian-client.js +92 -9
  43. package/src/health.js +99 -8
  44. package/src/managed-process.js +197 -16
  45. package/src/process-guardian.js +212 -22
  46. package/src/process-memory.js +62 -2
  47. package/src/process-order.js +26 -0
  48. package/src/release-group.js +452 -39
  49. package/tensorbuzz.yml +20 -1
  50. package/test/completion.test.js +5 -2
  51. package/test/compose-config.test.js +160 -0
  52. package/test/compose-docker.test.js +297 -0
  53. package/test/compose-unit.test.js +415 -0
  54. package/test/config-validation.test.js +135 -5
  55. package/test/control-protocol.test.js +22 -0
  56. package/test/drain-policy.test.js +328 -0
  57. package/test/fixtures/deferred-boot-app.js +106 -0
  58. package/test/fixtures/gated-app.js +50 -0
  59. package/test/guardian-client.test.js +149 -5
  60. package/test/health.test.js +74 -1
  61. package/test/managed-process.test.js +260 -1
  62. package/test/owner-recovery.test.js +151 -20
  63. package/test/owner-replacement.test.js +176 -15
  64. package/test/process-memory.test.js +47 -1
  65. package/test/release-group.test.js +447 -2
  66. package/test/rollbridge.test.js +774 -72
  67. package/test/shutdown-completion.test.js +119 -0
  68. package/test/shutdown-retirement-bound.test.js +162 -0
  69. package/test/support/process.js +3 -1
  70. package/test/tensorbuzz-config.test.js +228 -0
@@ -4,14 +4,21 @@ import fsSync from "node:fs"
4
4
  import fs from "node:fs/promises"
5
5
  import net from "node:net"
6
6
  import crypto from "node:crypto"
7
+ import {createRequire} from "node:module"
7
8
  import {spawn} from "node:child_process"
8
9
  import {isDeepStrictEqual} from "node:util"
9
10
  import path from "node:path"
10
11
  import GuardianClient from "./guardian-client.js"
11
- import ManagedProcess from "./managed-process.js"
12
+ import ManagedProcess, {childRecordIsGone} from "./managed-process.js"
12
13
  import {writeState} from "./state-store.js"
13
14
 
14
15
  const OWNER_RESTART_RETRY_MS = 1000
16
+ const commonJsRequire = createRequire(import.meta.url)
17
+ const GUARDIAN_VERSION = /** @type {{version: string}} */ (commonJsRequire("../package.json")).version
18
+ // An owner mutation is deploy/stop/restart/rollback/recovery work. A retained
19
+ // generation may legitimately drain for hours, but a mutation that holds the
20
+ // fence that long is stuck, so the lease is bounded and released on expiry.
21
+ const OWNER_MUTATION_LEASE_MS = 900000
15
22
 
16
23
  /** @typedef {import("node:child_process").ChildProcess["signalCode"]} ChildExitSignal */
17
24
 
@@ -24,7 +31,7 @@ const OWNER_RESTART_RETRY_MS = 1000
24
31
  * @property {number} id - Request id.
25
32
  * @property {string} [key] - Stable process key.
26
33
  * @property {boolean} [localSource] - Whether the retired daemon physically owns the published source.
27
- * @property {{timeoutMs?: number}} [options] - Stop options.
34
+ * @property {{retirementTimeoutMs?: number, timeoutMs?: number}} [options] - Stop options.
28
35
  * @property {string} [previousProvenance] - Expected current provenance.
29
36
  * @property {string} [provenance] - New or registered provenance.
30
37
  * @property {string} [releaseId] - Retained release identity.
@@ -32,7 +39,9 @@ const OWNER_RESTART_RETRY_MS = 1000
32
39
  * @property {import("./json.js").JsonValue} [ownerState] - Private owner transfer state.
33
40
  * @property {string} [replacementId] - Prepared replacement transaction id.
34
41
  * @property {string} [mutationId] - Owner mutation lease id.
42
+ * @property {number} [mutationTimeoutMs] - Bound for the owner mutation lease.
35
43
  * @property {string} [operation] - Owner mutation diagnostic name.
44
+ * @property {string} [waitingFor] - What the held owner mutation is blocked on.
36
45
  * @property {number} [ownerPid] - Exact claimant process PID.
37
46
  * @property {string} [configDigest] - Exact committed owner configuration digest.
38
47
  * @property {number} [pid] - Exact live process PID to re-adopt.
@@ -106,6 +115,19 @@ const legacyKeys = legacyGuardian ? committedOwnerProcessKeys(ownerState) : new
106
115
  let ownerMutationClient
107
116
  /** @type {string | undefined} */
108
117
  let ownerMutationId
118
+ /** @type {string | undefined} */
119
+ let ownerMutationOperation
120
+ /** @type {string | undefined} */
121
+ let ownerMutationWaitingFor
122
+ /** @type {number | undefined} */
123
+ let ownerMutationPid
124
+ /** @type {number | undefined} */
125
+ let ownerMutationStartedAt
126
+ let ownerMutationLeaseMs = OWNER_MUTATION_LEASE_MS
127
+ /** @type {ReturnType<typeof setTimeout> | undefined} */
128
+ let ownerMutationLeaseTimer
129
+ /** @type {{elapsedMs: number, leaseMs: number, mutationId: string, operation: string} | undefined} */
130
+ let ownerMutationExpired
109
131
  /** @type {net.Socket | undefined} */
110
132
  let retiringClient
111
133
  /** @type {string | undefined} */
@@ -157,10 +179,7 @@ const server = net.createServer((socket) => {
157
179
  clearTimeout(waiter.timer)
158
180
  waiter.reject(new Error("Owner claimant disconnected"))
159
181
  }
160
- if (ownerMutationClient === socket && ownerClient !== socket) {
161
- ownerMutationClient = undefined
162
- ownerMutationId = undefined
163
- }
182
+ if (ownerMutationClient === socket) clearOwnerMutationLease()
164
183
  if (controlLessRetirement) {
165
184
  const preCommitHandoff = replacementClient && ownerClient === socket && retiringReplacementId === replacementId
166
185
 
@@ -262,13 +281,14 @@ async function handleLine(socket, line) {
262
281
  async function execute(request, socket) {
263
282
  if (shuttingDown) throw new Error("Process guardian is shutting down")
264
283
 
265
- if (request.command === "capabilities") return {daemonRecovery: 1, generationReactivation: 1}
284
+ if (request.command === "capabilities") return {daemonRecovery: 1, generationReactivation: 1, guardianVersion: GUARDIAN_VERSION}
266
285
 
267
286
  if (request.command === "owner-replacement-capabilities") {
268
287
  return {commands: ["commit-retired-owner-replacement"], protocol: "owner-replacement", version: 1}
269
288
  }
270
289
 
271
290
  if (request.command === "claim-owner") {
291
+ reclaimDeadOwnerMutation()
272
292
  if (retirementFailed) throw new Error("Guardian owner replacement is fenced after incomplete retired-listener state transfer")
273
293
  if (ownerState !== undefined && !isDeepStrictEqual(request.authority, ownerAuthority(ownerState))) {
274
294
  const waitsForPreparedAuthority = Boolean(ownerClient && replacementOwnerState && isDeepStrictEqual(request.authority, replacementAuthority))
@@ -304,8 +324,7 @@ async function execute(request, socket) {
304
324
  rememberDisconnectedOwnerSuccessors(socket)
305
325
  ownerClient = undefined
306
326
  ownerClientPid = undefined
307
- ownerMutationClient = undefined
308
- ownerMutationId = undefined
327
+ clearOwnerMutationLease()
309
328
  ownerRevision += 1
310
329
  grantNextOwner()
311
330
  return {retired: true}
@@ -397,29 +416,54 @@ async function execute(request, socket) {
397
416
  return {committedReplacementId: committedReplacementId ?? null, ownerClaimed: Boolean(ownerClient), retirementFailed, retirementPending: Boolean(retiringClient), retirementReady: retiringListenerReady}
398
417
  }
399
418
 
419
+ if (request.command === "guardian-status") return guardianStatus()
420
+
400
421
  if (request.command === "begin-owner-mutation") {
401
422
  requireOwner(socket, request.command)
423
+ reclaimDeadOwnerMutation()
402
424
  if (replacementClient) throw new Error(`Owner mutation ${request.operation || "operation"} is fenced while an owner replacement is prepared`)
403
- if (ownerMutationClient) throw new Error("Another owner mutation is already in progress")
425
+ const leaseMs = request.mutationTimeoutMs === undefined ? OWNER_MUTATION_LEASE_MS : request.mutationTimeoutMs
426
+
427
+ if (!Number.isInteger(leaseMs) || leaseMs <= 0) throw new Error("Owner mutation lease bound must be a positive number of milliseconds")
428
+ if (ownerMutationClient) throw new Error(`Another owner mutation is already in progress: ${describeOwnerMutationLease()}; refused ${request.operation || "operation"}`)
404
429
  ownerMutationClient = socket
405
430
  ownerMutationId = crypto.randomUUID()
406
- return {mutationId: ownerMutationId}
431
+ ownerMutationOperation = request.operation
432
+ ownerMutationWaitingFor = request.waitingFor
433
+ ownerMutationPid = request.ownerPid ?? ownerClientPid
434
+ ownerMutationStartedAt = Date.now()
435
+ ownerMutationLeaseMs = leaseMs
436
+ armOwnerMutationLease(ownerMutationId)
437
+ return {leaseMs, mutationId: ownerMutationId}
407
438
  }
408
439
 
409
440
  if (request.command === "end-owner-mutation") {
410
441
  requireOwner(socket, request.command)
442
+ if (ownerMutationExpired && request.mutationId === ownerMutationExpired.mutationId) {
443
+ const expired = ownerMutationExpired
444
+
445
+ ownerMutationExpired = undefined
446
+ throw new Error(`Owner mutation ${expired.operation} exceeded its ${expired.leaseMs} ms lease and was released after ${expired.elapsedMs} ms; the mutation failed and its lease is gone`)
447
+ }
411
448
  if (ownerMutationClient !== socket || request.mutationId !== ownerMutationId) throw new Error("Owner mutation lease does not match the active mutation")
412
- ownerMutationClient = undefined
413
- ownerMutationId = undefined
449
+ clearOwnerMutationLease()
414
450
  return {ended: true}
415
451
  }
416
452
 
453
+ if (request.command === "report-owner-mutation-wait") {
454
+ requireOwner(socket, request.command)
455
+ if (ownerMutationClient !== socket || request.mutationId !== ownerMutationId) throw new Error("Owner mutation lease does not match the active mutation")
456
+ if (typeof request.waitingFor !== "string" || !request.waitingFor.trim()) throw new Error("Owner mutation wait report requires a waitingFor description")
457
+ ownerMutationWaitingFor = request.waitingFor.trim()
458
+ return {reported: true}
459
+ }
460
+
417
461
  if (request.command === "prepare-owner-replacement") {
418
462
  if (retirementFailed) throw new Error("Guardian owner replacement is fenced after incomplete retired-listener state transfer")
419
463
  if (retiringClient) throw new Error("Guardian owner replacement cannot prepare while listener retirement is pending")
420
464
  if (socket === ownerClient) throw new Error("Committed owner cannot prepare itself as a replacement")
421
465
  if (replacementClient && replacementClient !== socket) throw new Error("Another owner replacement candidate is already prepared")
422
- if (ownerMutationClient) throw new Error("Owner replacement cannot prepare while an owner mutation is in progress")
466
+ if (ownerMutationClient) throw new Error(`Owner replacement cannot prepare while an owner mutation is in progress: ${describeOwnerMutationLease()}`)
423
467
  if (!ownerState) throw new Error("Committed owner has not published transferable state")
424
468
  const currentAuthority = ownerAuthority(ownerState)
425
469
  const resumesCommittedAuthority = !ownerClient && isDeepStrictEqual(currentAuthority, request.nextAuthority)
@@ -448,7 +492,7 @@ async function execute(request, socket) {
448
492
 
449
493
  if (request.command === "stage-owner-replacement") {
450
494
  requireReplacement(socket, request)
451
- if (replacementRevision !== ownerRevision) throw new Error("Guardian owner state changed after prepare; abort the stale candidate before preparing again")
495
+ if (replacementRevision !== ownerRevision) throw new Error(`Guardian owner state changed after prepare (prepared revision ${replacementRevision}, current revision ${ownerRevision}); abort the stale candidate before preparing again`)
452
496
  if (request.ownerState === undefined || !isDeepStrictEqual(ownerAuthority(request.ownerState), replacementAuthority)) {
453
497
  throw new Error("Prepared replacement state does not match the requested authority")
454
498
  }
@@ -579,7 +623,7 @@ async function execute(request, socket) {
579
623
  requireOwner(socket, request.command)
580
624
  shuttingDown = true
581
625
  try {
582
- const results = await Promise.allSettled([...processes.values()].map((entry) => entry.process.stop()))
626
+ const results = await Promise.allSettled([...processes.values()].map((entry) => entry.process.stop(request.options)))
583
627
  const errors = results.filter((result) => result.status === "rejected").map((result) => result.reason)
584
628
 
585
629
  if (errors.length > 0) throw new AggregateError(errors, `Guardian failed to stop ${errors.length} owned process${errors.length === 1 ? "" : "es"}: ${errors.map((error) => errorMessage(error instanceof Error ? error : String(error))).join("; ")}`)
@@ -616,8 +660,17 @@ async function execute(request, socket) {
616
660
  const existing = processes.get(request.key)
617
661
 
618
662
  if (existing) {
619
- if (existing.provenance !== request.provenance) throw new Error(`Guardian provenance mismatch for ${request.key}`)
620
- return existing.process.status()
663
+ if (existing.provenance === request.provenance) return existing.process.status()
664
+ // A provenance mismatch only fences a process that still exists. When the registered
665
+ // child is conclusively gone (stopped or failed with no live pid), the old record is
666
+ // residue of a dead generation, and demanding its provenance wedges every later deploy
667
+ // against a process that no longer exists (TensorBuzz production, STC-695). The owner
668
+ // may re-register the key under the new definition; live processes stay fail-closed.
669
+ // A group that outlived its tracked process is not gone: it still holds the key's work,
670
+ // so re-registering it would start a second copy of a generation that is still running.
671
+ if (!childRecordIsGone(existing.process.status())) throw new Error(`Guardian provenance mismatch for ${request.key}`)
672
+ requireOwner(socket, request.command)
673
+ processes.delete(request.key)
621
674
  }
622
675
 
623
676
  const recoversLegacyProcess = Boolean(legacyGuardian && legacyKeys.has(request.key))
@@ -720,6 +773,146 @@ async function execute(request, socket) {
720
773
  return status
721
774
  }
722
775
 
776
+ /**
777
+ * Names the mutation holding the owner-mutation fence: which mutation, which process,
778
+ * since when, and what it is blocked on. Refusals must be diagnosable without a live
779
+ * inspection of the daemon.
780
+ * @returns {string} Holder description used in refusal messages.
781
+ */
782
+ function describeOwnerMutationLease() {
783
+ const startedAt = ownerMutationStartedAt ?? 0
784
+ const elapsedSeconds = Math.max(0, Math.round((Date.now() - startedAt) / 1000))
785
+ const holderPid = ownerMutationPid ?? ownerClientPid
786
+
787
+ return [
788
+ `owner mutation "${ownerMutationOperation || "unnamed operation"}"`,
789
+ `mutation ${ownerMutationId || "unknown"}`,
790
+ `held by process ${holderPid ?? "unknown"}`,
791
+ `since ${new Date(startedAt).toISOString()}`,
792
+ `${elapsedSeconds}s elapsed`,
793
+ `waiting for ${ownerMutationWaitingFor || "the operation to report completion"}`
794
+ ].join("; ")
795
+ }
796
+
797
+ /**
798
+ * Reports the currently held owner-mutation lease.
799
+ * @returns {{elapsedMs: number, leaseMs: number, mutationId: string, operation: string, pid: number | null, startedAt: string, waitingFor: string} | null} Active lease, if any.
800
+ */
801
+ function activeOwnerMutationStatus() {
802
+ if (!ownerMutationClient || !ownerMutationId) return null
803
+
804
+ const startedAt = ownerMutationStartedAt ?? Date.now()
805
+
806
+ return {
807
+ elapsedMs: Math.max(0, Date.now() - startedAt),
808
+ leaseMs: ownerMutationLeaseMs,
809
+ mutationId: ownerMutationId,
810
+ operation: ownerMutationOperation || "unnamed operation",
811
+ pid: ownerMutationPid ?? ownerClientPid ?? null,
812
+ startedAt: new Date(startedAt).toISOString(),
813
+ waitingFor: ownerMutationWaitingFor || "the operation to report completion"
814
+ }
815
+ }
816
+
817
+ /**
818
+ * Reports guardian identity and owner-mutation diagnostics.
819
+ * @returns {{guardianPid: number, ownerMutation: ReturnType<typeof activeOwnerMutationStatus>, version: string}} Guardian diagnostics.
820
+ */
821
+ function guardianStatus() {
822
+ return {guardianPid: process.pid, ownerMutation: activeOwnerMutationStatus(), version: GUARDIAN_VERSION}
823
+ }
824
+
825
+ /** @returns {boolean} Whether the recorded owner-mutation holder is no longer alive. */
826
+ function reclaimDeadOwnerMutation() {
827
+ if (!ownerMutationClient || !ownerMutationPid || isProcessAlive(ownerMutationPid)) return false
828
+
829
+ const mutation = activeOwnerMutationStatus()
830
+ const deadOwnerSocket = ownerClient === ownerMutationClient && ownerClientPid === ownerMutationPid ? ownerClient : undefined
831
+ const diagnostic = {
832
+ code: "OWNER_MUTATION_RECLAIMED",
833
+ event: "owner-mutation-reclaimed",
834
+ mutationId: ownerMutationId,
835
+ operation: mutation?.operation || "unnamed operation",
836
+ pid: ownerMutationPid,
837
+ reason: "owner mutation holder process is no longer alive"
838
+ }
839
+
840
+ process.stderr.write(`${JSON.stringify({at: new Date().toISOString(), ...diagnostic})}\n`)
841
+ broadcast(diagnostic)
842
+ clearOwnerMutationLease()
843
+ if (deadOwnerSocket) releaseClosedOwner(deadOwnerSocket)
844
+ return true
845
+ }
846
+
847
+ /**
848
+ * Checks whether a process still exists without signaling it.
849
+ * @param {number} pid - Process id.
850
+ * @returns {boolean} Whether the process exists.
851
+ */
852
+ function isProcessAlive(pid) {
853
+ try {
854
+ process.kill(pid, 0)
855
+ return true
856
+ } catch (error) {
857
+ return Boolean(error && typeof error === "object" && "code" in error && error.code === "EPERM")
858
+ }
859
+ }
860
+
861
+ /** @returns {void} */
862
+ function clearOwnerMutationLease() {
863
+ if (ownerMutationLeaseTimer) clearTimeout(ownerMutationLeaseTimer)
864
+ ownerMutationLeaseTimer = undefined
865
+ ownerMutationClient = undefined
866
+ ownerMutationId = undefined
867
+ ownerMutationOperation = undefined
868
+ ownerMutationWaitingFor = undefined
869
+ ownerMutationPid = undefined
870
+ ownerMutationStartedAt = undefined
871
+ ownerMutationLeaseMs = OWNER_MUTATION_LEASE_MS
872
+ }
873
+
874
+ /**
875
+ * Arms the lease bound for one mutation.
876
+ * @param {string} mutationId - Lease id the timer belongs to.
877
+ * @returns {void}
878
+ */
879
+ function armOwnerMutationLease(mutationId) {
880
+ if (ownerMutationLeaseTimer) clearTimeout(ownerMutationLeaseTimer)
881
+ ownerMutationLeaseTimer = setTimeout(() => expireOwnerMutationLease(mutationId), ownerMutationLeaseMs)
882
+ ownerMutationLeaseTimer.unref()
883
+ }
884
+
885
+ /**
886
+ * A mutation that outruns its lease is reported and released instead of holding the fence
887
+ * forever, so a stuck mutation cannot lock out every later deploy. The holder learns the
888
+ * lease is gone when it tries to end it.
889
+ * @param {string} mutationId - Lease id the armed timer belongs to.
890
+ * @returns {void}
891
+ */
892
+ function expireOwnerMutationLease(mutationId) {
893
+ ownerMutationLeaseTimer = undefined
894
+ if (!ownerMutationClient || ownerMutationId !== mutationId) return
895
+ const operation = ownerMutationOperation || "unnamed operation"
896
+ const leaseMs = ownerMutationLeaseMs
897
+ const elapsedMs = Math.max(0, Date.now() - (ownerMutationStartedAt ?? 0))
898
+ const diagnostic = {
899
+ code: "OWNER_MUTATION_LEASE_EXPIRED",
900
+ elapsedMs,
901
+ event: "owner-mutation-expired",
902
+ leaseMs,
903
+ mutationId,
904
+ operation,
905
+ pid: ownerMutationPid ?? ownerClientPid,
906
+ waitingFor: ownerMutationWaitingFor || "the operation to report completion"
907
+ }
908
+
909
+ process.stderr.write(`${JSON.stringify({at: new Date().toISOString(), ...diagnostic})}\n`)
910
+ if (!ownerMutationClient.destroyed) ownerMutationClient.write(`${JSON.stringify(diagnostic)}\n`)
911
+ broadcast(diagnostic)
912
+ ownerMutationExpired = {elapsedMs, leaseMs, mutationId, operation}
913
+ clearOwnerMutationLease()
914
+ }
915
+
723
916
  /**
724
917
  * @param {net.Socket} socket - Requesting client.
725
918
  * @param {string} command - Command name.
@@ -740,10 +933,7 @@ function releaseClosedOwner(socket) {
740
933
  rememberDisconnectedOwnerSuccessors(socket)
741
934
  ownerClient = undefined
742
935
  ownerClientPid = undefined
743
- if (ownerMutationClient === socket) {
744
- ownerMutationClient = undefined
745
- ownerMutationId = undefined
746
- }
936
+ if (ownerMutationClient === socket) clearOwnerMutationLease()
747
937
  if (replacementClient && replacementOwnerState) commitReplacement()
748
938
  else {
749
939
  if (incompleteRestartChild) {
@@ -4,8 +4,14 @@ import fs from "node:fs"
4
4
 
5
5
  /**
6
6
  * @typedef {{command: string, pid: number, rssBytes: number | undefined}} ProcessGroupMember
7
+ * @typedef {{cpuMs: number | undefined, liveMemberCount: number, memberCount: number}} ProcessGroupSample
7
8
  */
8
9
 
10
+ // Linux reports /proc/<pid>/stat CPU time in clock ticks and keeps USER_HZ at 100 for
11
+ // userspace, so ticks * 10 / 1 is milliseconds. This is a monotonic per-process counter,
12
+ // not a rate: two reads of it measure whether a process did any work between them.
13
+ const CLOCK_TICKS_PER_MS = 10
14
+
9
15
  /**
10
16
  * Lists the members of a managed process group with each member's resident memory.
11
17
  * Rollbridge spawns each process detached, so the spawned pid is the process-group
@@ -108,6 +114,51 @@ export function measureProcessGroupRssBytes(pgid) {
108
114
  return measured.reduce((total, member) => total + (member.rssBytes ?? 0), 0)
109
115
  }
110
116
 
117
+ /**
118
+ * Samples one managed process group for drain liveness: how much CPU its members have
119
+ * consumed in total, and how many of them can still run.
120
+ *
121
+ * CPU is the sum of every member's cumulative user+system time, defunct members included:
122
+ * a member that exits takes its consumed time out of the sum, so the difference between two
123
+ * samples is non-zero whenever the group changed at all — new work or a departing member.
124
+ * Only an exactly unchanged difference says the group did nothing between the two reads.
125
+ * @param {number} pgid - Process-group id (the detached spawn's pid).
126
+ * @param {string} [procPath] - Procfs root (overridable for deterministic tests).
127
+ * @returns {ProcessGroupSample | undefined} The sample, or undefined only when procfs cannot be
128
+ * read at all. A group with no members left is a sample of zero, not a missing measurement.
129
+ */
130
+ export function sampleProcessGroup(pgid, procPath = "/proc") {
131
+ /** @type {string[]} */
132
+ let entries
133
+
134
+ try {
135
+ entries = fs.readdirSync(procPath)
136
+ } catch {
137
+ return undefined
138
+ }
139
+
140
+ let cpuMs = 0
141
+ let liveMemberCount = 0
142
+ let memberCount = 0
143
+ let measurable = true
144
+
145
+ for (const entry of entries) {
146
+ if (!/^\d+$/.test(entry)) continue
147
+
148
+ const stat = processStat(entry, procPath)
149
+
150
+ if (stat?.pgrp !== pgid) continue
151
+
152
+ memberCount += 1
153
+ if (stat.state !== "Z" && stat.state !== "X") liveMemberCount += 1
154
+
155
+ if (stat.cpuMs === undefined) measurable = false
156
+ else cpuMs += stat.cpuMs
157
+ }
158
+
159
+ return {cpuMs: measurable ? cpuMs : undefined, liveMemberCount, memberCount}
160
+ }
161
+
111
162
  /**
112
163
  * @param {string} pid - Process id.
113
164
  * @returns {string} The process command name (`/proc/<pid>/comm`), or "" when unavailable.
@@ -131,7 +182,7 @@ function processGroupId(pid) {
131
182
  /**
132
183
  * @param {string} pid - Process id.
133
184
  * @param {string} procPath - Procfs root.
134
- * @returns {{pgrp: number, state: string} | undefined} Parsed process state and group.
185
+ * @returns {{cpuMs: number | undefined, pgrp: number, state: string} | undefined} Parsed process state, group, and cumulative CPU.
135
186
  */
136
187
  function processStat(pid, procPath) {
137
188
  let stat
@@ -151,7 +202,16 @@ function processStat(pid, procPath) {
151
202
  const fields = stat.slice(commEnd + 2).split(" ")
152
203
  const pgrp = Number(fields[2])
153
204
 
154
- return Number.isInteger(pgrp) ? {pgrp, state: fields[0]} : undefined
205
+ if (!Number.isInteger(pgrp)) return undefined
206
+
207
+ // utime and stime are fields 14 and 15 of stat, i.e. index 11 and 12 after the comm field.
208
+ const utimeTicks = Number(fields[11])
209
+ const stimeTicks = Number(fields[12])
210
+ const cpuMs = Number.isFinite(utimeTicks) && Number.isFinite(stimeTicks)
211
+ ? (utimeTicks + stimeTicks) * CLOCK_TICKS_PER_MS
212
+ : undefined
213
+
214
+ return {cpuMs, pgrp, state: fields[0]}
155
215
  }
156
216
 
157
217
  /**
@@ -0,0 +1,26 @@
1
+ // @ts-check
2
+
3
+ /**
4
+ * @typedef {import("./config.js").ProcessConfig} ProcessConfig
5
+ */
6
+
7
+ const POLICY_START_RANK = /** @type {Record<ProcessConfig["policy"], number>} */ ({companion: 1, proxied: 2, service: 0, singleton: 3})
8
+
9
+ /**
10
+ * Order in which Rollbridge starts processes, which is also the only order a `dependsOn`
11
+ * edge may point backwards in: daemon-wide persistent services are started before every
12
+ * release generation, then handoff services, companions, and the proxied process last.
13
+ * Config order is kept inside each group.
14
+ * @param {ProcessConfig[]} processes - Normalized processes.
15
+ * @param {{includePersistentServices?: boolean}} [options] - Whether to include the
16
+ * daemon-wide persistent services that a release group never starts itself.
17
+ * @returns {ProcessConfig[]} Ordered process configs.
18
+ */
19
+ export function processStartOrder(processes, {includePersistentServices = false} = {}) {
20
+ const persistentServices = includePersistentServices
21
+ ? processes.filter((processConfig) => processConfig.policy === "service" && processConfig.deployStrategy === "persistent")
22
+ : []
23
+ const generationProcesses = processes.filter((processConfig) => processConfig.policy !== "singleton" && (processConfig.policy !== "service" || processConfig.deployStrategy === "handoff"))
24
+
25
+ return [...persistentServices, ...generationProcesses.sort((left, right) => POLICY_START_RANK[left.policy] - POLICY_START_RANK[right.policy])]
26
+ }