rollbridge 0.1.68 → 0.1.70

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -22,7 +22,7 @@ const ESCALATION_KILL_CONFIRM_MS = 5000
22
22
  * @typedef {import("node:child_process").ChildProcess["signalCode"]} ProcessExitSignal
23
23
  * @typedef {{at: string, line: string, stream: "stdout" | "stderr"}} ManagedProcessLog
24
24
  * @typedef {import("./config.js").StopTimeoutMs} StopTimeoutMs
25
- * @typedef {{command: string, cwd: string | undefined, env: Record<string, string | undefined>, lifecycle: import("./config.js").LifecycleConfig, logger: (message: string, data?: Record<string, import("./json.js").JsonValue>) => void, memory: import("./config.js").MemoryConfig | undefined, outputLines: number, restart: import("./config.js").RestartConfig, restartDelayMs: number, shouldRestart: () => boolean, stopSignal: string, stopTimeoutMs: StopTimeoutMs}} ManagedProcessDefinition
25
+ * @typedef {{command: string, cwd: string | undefined, env: Record<string, string | undefined>, lifecycle: import("./config.js").LifecycleConfig, logger: (message: string, data?: Record<string, import("./json.js").JsonValue>) => void, memory: import("./config.js").MemoryConfig | undefined, outputLines: number, restart: import("./config.js").RestartConfig, restartDelayMs: number, shouldRestart: () => boolean, shouldStop?: () => boolean, stopSignal: string, stopTimeoutMs: StopTimeoutMs}} ManagedProcessDefinition
26
26
  * @typedef {{liveMemberCount: number | undefined, pgid: number}} HeldProcessGroup
27
27
  * @typedef {{children: import("./process-memory.js").ProcessGroupMember[], command: string, cwd: string | undefined, exitCode: number | null | undefined, exitSignal: ProcessExitSignal | undefined, heldProcessGroup: HeldProcessGroup | undefined, id: string, lastMemoryRestartAt: string | undefined, lastStartReason: ManagedProcessStartReason | undefined, lifecycleRole?: LifecycleRole, logs: ManagedProcessLog[], memoryRestarts: number, pid: number | undefined, restarts: number, retirementTimedOut: boolean, rssBytes: number | undefined, startedAt: string | undefined, state: ManagedProcessState, uptimeMs: number | undefined}} ManagedProcessStatus
28
28
  */
@@ -41,10 +41,11 @@ export default class ManagedProcess extends EventEmitter {
41
41
  * @param {import("./config.js").RestartConfig} [args.restart] - Restart policy (defaults to unlimited restarts with a constant delay).
42
42
  * @param {number} args.restartDelayMs - Restart delay.
43
43
  * @param {() => boolean} args.shouldRestart - Restart policy callback.
44
+ * @param {() => boolean} [args.shouldStop] - Whether an unexpected exit is terminal because its dependency retired.
44
45
  * @param {string} [args.stopSignal] - Signal sent to gracefully stop the process (default "SIGTERM").
45
46
  * @param {StopTimeoutMs} args.stopTimeoutMs - Stop timeout.
46
47
  */
47
- constructor({command, cwd, env, id, lifecycle = {drainTimeoutMs: 0}, logger, memory, outputLines, restart = {backoffFactor: 1, maxDelayMs: 0, maxRestarts: undefined, windowMs: 0}, restartDelayMs, shouldRestart, stopSignal = "SIGTERM", stopTimeoutMs}) {
48
+ constructor({command, cwd, env, id, lifecycle = {drainTimeoutMs: 0}, logger, memory, outputLines, restart = {backoffFactor: 1, maxDelayMs: 0, maxRestarts: undefined, windowMs: 0}, restartDelayMs, shouldRestart, shouldStop = () => false, stopSignal = "SIGTERM", stopTimeoutMs}) {
48
49
  super()
49
50
 
50
51
  this.command = command
@@ -58,6 +59,7 @@ export default class ManagedProcess extends EventEmitter {
58
59
  this.restart = restart
59
60
  this.restartDelayMs = restartDelayMs
60
61
  this.shouldRestart = shouldRestart
62
+ this.shouldStop = shouldStop
61
63
  this.stopSignal = stopSignal
62
64
  this.stopTimeoutMs = stopTimeoutMs
63
65
  this.state = /** @type {ManagedProcessState} */ ("stopped")
@@ -207,6 +209,7 @@ export default class ManagedProcess extends EventEmitter {
207
209
  this.restart = definition.restart
208
210
  this.restartDelayMs = definition.restartDelayMs
209
211
  this.shouldRestart = definition.shouldRestart
212
+ this.shouldStop = definition.shouldStop ?? (() => false)
210
213
  this.stopSignal = definition.stopSignal
211
214
  this.stopTimeoutMs = definition.stopTimeoutMs
212
215
  }
@@ -265,12 +268,13 @@ export default class ManagedProcess extends EventEmitter {
265
268
  this.rssBytes = undefined
266
269
  this.children = []
267
270
  this.clearMemoryMonitor()
268
- this.state = wasIntentional ? "stopped" : "failed"
271
+ const terminalExit = !wasIntentional && this.shouldStop()
272
+ this.state = wasIntentional || terminalExit ? "stopped" : "failed"
269
273
 
270
274
  this.logger("process exited", {code, id: this.id, signal})
271
275
  this.emit("exit", {code, signal})
272
276
 
273
- if (!wasIntentional && this.shouldRestart()) {
277
+ if (!wasIntentional && !terminalExit && this.shouldRestart()) {
274
278
  this.scheduleRestart()
275
279
  }
276
280
  }
@@ -507,6 +511,7 @@ export default class ManagedProcess extends EventEmitter {
507
511
  async performStop(options) {
508
512
  const pgid = this.child?.pid ?? this.pid
509
513
  const exitPromise = this.exitPromise
514
+ const retirementDeadline = options.retirementTimeoutMs === undefined ? undefined : Date.now() + options.retirementTimeoutMs
510
515
  await this.quiesce()
511
516
 
512
517
  if (!pgid) {
@@ -528,7 +533,6 @@ export default class ManagedProcess extends EventEmitter {
528
533
  // 3. Stop whatever is still running, then SIGKILL if it outlasts the graceful window.
529
534
  if (this.processGroupExists(pgid)) {
530
535
  const timeoutMs = options.timeoutMs ?? this.stopTimeoutMs
531
- const retirementDeadline = options.retirementTimeoutMs === undefined ? undefined : Date.now() + options.retirementTimeoutMs
532
536
  const gracefulDeadline = timeoutMs === "indefinite" ? undefined : Date.now() + timeoutMs
533
537
  // A retirement budget is the caller's decision that this process must not be waited for and
534
538
  // must not be killed, so it caps the graceful window rather than only standing in for a
@@ -550,6 +554,11 @@ export default class ManagedProcess extends EventEmitter {
550
554
  if (!exited && retirementBound) {
551
555
  this.retirementTimedOut = true
552
556
  this.markGroupOutlivedStop(pgid)
557
+ // The tracked process has completed its supervised stop even though its process group is
558
+ // still draining. Keep the residue in heldProcessGroup rather than leaving the process
559
+ // permanently quiesced, which would make a drained generation look active to restore
560
+ // admission and keep the owner state from describing the actual stopped generation.
561
+ this.state = "stopped"
553
562
  this.logger("process retirement wait timed out; leaving process alive", {id: this.id, pid: pgid, timeoutMs: options.retirementTimeoutMs})
554
563
  return
555
564
  }
@@ -4,6 +4,7 @@ import fsSync from "node:fs"
4
4
  import fs from "node:fs/promises"
5
5
  import net from "node:net"
6
6
  import crypto from "node:crypto"
7
+ import {createRequire} from "node:module"
7
8
  import {spawn} from "node:child_process"
8
9
  import {isDeepStrictEqual} from "node:util"
9
10
  import path from "node:path"
@@ -12,6 +13,8 @@ import ManagedProcess, {childRecordIsGone} from "./managed-process.js"
12
13
  import {writeState} from "./state-store.js"
13
14
 
14
15
  const OWNER_RESTART_RETRY_MS = 1000
16
+ const commonJsRequire = createRequire(import.meta.url)
17
+ const GUARDIAN_VERSION = /** @type {{version: string}} */ (commonJsRequire("../package.json")).version
15
18
  // An owner mutation is deploy/stop/restart/rollback/recovery work. A retained
16
19
  // generation may legitimately drain for hours, but a mutation that holds the
17
20
  // fence that long is stuck, so the lease is bounded and released on expiry.
@@ -28,7 +31,7 @@ const OWNER_MUTATION_LEASE_MS = 900000
28
31
  * @property {number} id - Request id.
29
32
  * @property {string} [key] - Stable process key.
30
33
  * @property {boolean} [localSource] - Whether the retired daemon physically owns the published source.
31
- * @property {{timeoutMs?: number}} [options] - Stop options.
34
+ * @property {{retirementTimeoutMs?: number, timeoutMs?: number}} [options] - Stop options.
32
35
  * @property {string} [previousProvenance] - Expected current provenance.
33
36
  * @property {string} [provenance] - New or registered provenance.
34
37
  * @property {string} [releaseId] - Retained release identity.
@@ -173,7 +176,7 @@ const server = net.createServer((socket) => {
173
176
  clearTimeout(waiter.timer)
174
177
  waiter.reject(new Error("Owner claimant disconnected"))
175
178
  }
176
- if (ownerMutationClient === socket && ownerClient !== socket) clearOwnerMutationLease()
179
+ if (ownerMutationClient === socket) clearOwnerMutationLease()
177
180
  if (controlLessRetirement) {
178
181
  const preCommitHandoff = replacementClient && ownerClient === socket && retiringReplacementId === replacementId
179
182
 
@@ -275,13 +278,14 @@ async function handleLine(socket, line) {
275
278
  async function execute(request, socket) {
276
279
  if (shuttingDown) throw new Error("Process guardian is shutting down")
277
280
 
278
- if (request.command === "capabilities") return {daemonRecovery: 1, generationReactivation: 1}
281
+ if (request.command === "capabilities") return {daemonRecovery: 1, generationReactivation: 1, guardianVersion: GUARDIAN_VERSION}
279
282
 
280
283
  if (request.command === "owner-replacement-capabilities") {
281
284
  return {commands: ["commit-retired-owner-replacement"], protocol: "owner-replacement", version: 1}
282
285
  }
283
286
 
284
287
  if (request.command === "claim-owner") {
288
+ reclaimDeadOwnerMutation()
285
289
  if (retirementFailed) throw new Error("Guardian owner replacement is fenced after incomplete retired-listener state transfer")
286
290
  if (ownerState !== undefined && !isDeepStrictEqual(request.authority, ownerAuthority(ownerState))) {
287
291
  const waitsForPreparedAuthority = Boolean(ownerClient && replacementOwnerState && isDeepStrictEqual(request.authority, replacementAuthority))
@@ -409,8 +413,11 @@ async function execute(request, socket) {
409
413
  return {committedReplacementId: committedReplacementId ?? null, ownerClaimed: Boolean(ownerClient), retirementFailed, retirementPending: Boolean(retiringClient), retirementReady: retiringListenerReady}
410
414
  }
411
415
 
416
+ if (request.command === "guardian-status") return guardianStatus()
417
+
412
418
  if (request.command === "begin-owner-mutation") {
413
419
  requireOwner(socket, request.command)
420
+ reclaimDeadOwnerMutation()
414
421
  if (replacementClient) throw new Error(`Owner mutation ${request.operation || "operation"} is fenced while an owner replacement is prepared`)
415
422
  const leaseMs = request.mutationTimeoutMs === undefined ? OWNER_MUTATION_LEASE_MS : request.mutationTimeoutMs
416
423
 
@@ -482,7 +489,7 @@ async function execute(request, socket) {
482
489
 
483
490
  if (request.command === "stage-owner-replacement") {
484
491
  requireReplacement(socket, request)
485
- if (replacementRevision !== ownerRevision) throw new Error("Guardian owner state changed after prepare; abort the stale candidate before preparing again")
492
+ if (replacementRevision !== ownerRevision) throw new Error(`Guardian owner state changed after prepare (prepared revision ${replacementRevision}, current revision ${ownerRevision}); abort the stale candidate before preparing again`)
486
493
  if (request.ownerState === undefined || !isDeepStrictEqual(ownerAuthority(request.ownerState), replacementAuthority)) {
487
494
  throw new Error("Prepared replacement state does not match the requested authority")
488
495
  }
@@ -613,7 +620,7 @@ async function execute(request, socket) {
613
620
  requireOwner(socket, request.command)
614
621
  shuttingDown = true
615
622
  try {
616
- const results = await Promise.allSettled([...processes.values()].map((entry) => entry.process.stop()))
623
+ const results = await Promise.allSettled([...processes.values()].map((entry) => entry.process.stop(request.options)))
617
624
  const errors = results.filter((result) => result.status === "rejected").map((result) => result.reason)
618
625
 
619
626
  if (errors.length > 0) throw new AggregateError(errors, `Guardian failed to stop ${errors.length} owned process${errors.length === 1 ? "" : "es"}: ${errors.map((error) => errorMessage(error instanceof Error ? error : String(error))).join("; ")}`)
@@ -772,6 +779,70 @@ function describeOwnerMutationLease() {
772
779
  ].join("; ")
773
780
  }
774
781
 
782
+ /**
783
+ * Reports the currently held owner-mutation lease.
784
+ * @returns {{elapsedMs: number, leaseMs: number, mutationId: string, operation: string, pid: number | null, startedAt: string, waitingFor: string} | null} Active lease, if any.
785
+ */
786
+ function activeOwnerMutationStatus() {
787
+ if (!ownerMutationClient || !ownerMutationId) return null
788
+
789
+ const startedAt = ownerMutationStartedAt ?? Date.now()
790
+
791
+ return {
792
+ elapsedMs: Math.max(0, Date.now() - startedAt),
793
+ leaseMs: ownerMutationLeaseMs,
794
+ mutationId: ownerMutationId,
795
+ operation: ownerMutationOperation || "unnamed operation",
796
+ pid: ownerMutationPid ?? ownerClientPid ?? null,
797
+ startedAt: new Date(startedAt).toISOString(),
798
+ waitingFor: ownerMutationWaitingFor || "the operation to report completion"
799
+ }
800
+ }
801
+
802
+ /**
803
+ * Reports guardian identity and owner-mutation diagnostics.
804
+ * @returns {{guardianPid: number, ownerMutation: ReturnType<typeof activeOwnerMutationStatus>, version: string}} Guardian diagnostics.
805
+ */
806
+ function guardianStatus() {
807
+ return {guardianPid: process.pid, ownerMutation: activeOwnerMutationStatus(), version: GUARDIAN_VERSION}
808
+ }
809
+
810
+ /** @returns {boolean} Whether the recorded owner-mutation holder is no longer alive. */
811
+ function reclaimDeadOwnerMutation() {
812
+ if (!ownerMutationClient || !ownerMutationPid || isProcessAlive(ownerMutationPid)) return false
813
+
814
+ const mutation = activeOwnerMutationStatus()
815
+ const deadOwnerSocket = ownerClient === ownerMutationClient && ownerClientPid === ownerMutationPid ? ownerClient : undefined
816
+ const diagnostic = {
817
+ code: "OWNER_MUTATION_RECLAIMED",
818
+ event: "owner-mutation-reclaimed",
819
+ mutationId: ownerMutationId,
820
+ operation: mutation?.operation || "unnamed operation",
821
+ pid: ownerMutationPid,
822
+ reason: "owner mutation holder process is no longer alive"
823
+ }
824
+
825
+ process.stderr.write(`${JSON.stringify({at: new Date().toISOString(), ...diagnostic})}\n`)
826
+ broadcast(diagnostic)
827
+ clearOwnerMutationLease()
828
+ if (deadOwnerSocket) releaseClosedOwner(deadOwnerSocket)
829
+ return true
830
+ }
831
+
832
+ /**
833
+ * Checks whether a process still exists without signaling it.
834
+ * @param {number} pid - Process id.
835
+ * @returns {boolean} Whether the process exists.
836
+ */
837
+ function isProcessAlive(pid) {
838
+ try {
839
+ process.kill(pid, 0)
840
+ return true
841
+ } catch (error) {
842
+ return Boolean(error && typeof error === "object" && "code" in error && error.code === "EPERM")
843
+ }
844
+ }
845
+
775
846
  /** @returns {void} */
776
847
  function clearOwnerMutationLease() {
777
848
  if (ownerMutationLeaseTimer) clearTimeout(ownerMutationLeaseTimer)
@@ -23,7 +23,7 @@ const DRAIN_SAMPLE_WINDOW_MS = 20000
23
23
  * @typedef {{cpuDeltaMs: number | undefined, reason: string, sampleWindowMs: number, status: import("./drain-policy.js").DrainProgress}} ReleaseDrainStatus
24
24
  * @typedef {{http: number, websocket: number}} ReleaseConnections
25
25
  * @typedef {{activatedAt: string | undefined, connectionCount: number, connections: ReleaseConnections, drain: ReleaseDrainStatus | undefined, drainStartedAt: string | undefined, ports: Record<string, number>, processes: import("./managed-process.js").ManagedProcessStatus[], releaseId: string, releasePath: string, retirementError: string | undefined, revision: string, state: ReleaseState, stoppedAt: string | undefined}} ReleaseStatus
26
- * @typedef {{count?: number, guardianKey?: string, index?: number, instanceId?: string, shouldRestart?: () => boolean}} BuildProcessOptions
26
+ * @typedef {{count?: number, guardianKey?: string, index?: number, instanceId?: string, shouldRestart?: () => boolean, shouldStop?: () => boolean}} BuildProcessOptions
27
27
  * @typedef {{deadlineMs: number, dependencyWaitMs: number, elapsedMs: number | undefined, failedProcessId: string, probes: number | undefined, stage: "dependency" | "health"}} StartupFailureDiagnostics
28
28
  */
29
29
 
@@ -507,6 +507,15 @@ export default class ReleaseGroup extends EventEmitter {
507
507
  this.clearDrainLivenessSamples()
508
508
  }
509
509
 
510
+ /** Stops only the proxied processes when a failed compensation cannot restore jobs. */
511
+ async stopProxyTraffic() {
512
+ const proxiedIds = new Set(this.config.processes.filter((processConfig) => processConfig.policy === "proxied").map((processConfig) => processConfig.id))
513
+
514
+ await Promise.allSettled([...this.processes.entries()]
515
+ .filter(([id]) => proxiedIds.has(id.replace(/#\d+$/, "")))
516
+ .map(([, processInstance]) => processInstance.stop()))
517
+ }
518
+
510
519
  /**
511
520
  * Restarts only the exact processes reconstructed for a committed generation.
512
521
  * The caller must prove the durable transition identity before using this path.
@@ -625,6 +634,10 @@ export default class ReleaseGroup extends EventEmitter {
625
634
  ...this.baseEnvironment(processConfig, {count, index}),
626
635
  ...renderedEnv
627
636
  }
637
+ const dependencyRetired = () => processConfig.dependsOn.some((dependencyId) => {
638
+ const dependency = this.processes.get(dependencyId)
639
+ return dependency?.lifecycleRole === "retired" || dependency?.status().lifecycleRole === "retired"
640
+ })
628
641
 
629
642
  return {
630
643
  command: renderTemplate(processConfig.command, context),
@@ -637,7 +650,15 @@ export default class ReleaseGroup extends EventEmitter {
637
650
  outputLines: processConfig.outputLines,
638
651
  restart: processConfig.restart,
639
652
  restartDelayMs: processConfig.restartDelayMs,
640
- shouldRestart: options.shouldRestart || (() => this.state === "active" || this.state === "starting"),
653
+ shouldRestart: options.shouldRestart || (() => {
654
+ if (this.state !== "active" && this.state !== "starting") return false
655
+
656
+ // A companion whose same-generation service dependency has retired can never
657
+ // satisfy its startup gate again. Treat that dependency lifecycle as terminal
658
+ // for this process so an exit does not become an endless supervisor restart loop.
659
+ return !dependencyRetired()
660
+ }),
661
+ shouldStop: options.shouldStop || dependencyRetired,
641
662
  stopSignal: processConfig.stopSignal,
642
663
  stopTimeoutMs: processConfig.gracefulStopMs
643
664
  }
@@ -894,7 +915,8 @@ export default class ReleaseGroup extends EventEmitter {
894
915
  return
895
916
  }
896
917
 
897
- await Promise.allSettled(connectionDependent.map((processInstance) => processInstance.stop()))
918
+ const retirementTimeoutMs = config.proxy.nonBlockingDrainTimeoutMs
919
+ await Promise.allSettled(connectionDependent.map((processInstance) => processInstance.stop({retirementTimeoutMs})))
898
920
  const nonBlockingResults = await Promise.allSettled(nonBlockingStops.map(({promise}) => promise))
899
921
  nonBlockingResults.forEach((result, index) => {
900
922
  if (result.status === "fulfilled" && this.processes.get(nonBlockingStops[index].id)?.retirementTimedOut) {
@@ -907,7 +929,7 @@ export default class ReleaseGroup extends EventEmitter {
907
929
  })
908
930
  }
909
931
  })
910
- await Promise.allSettled(handoffServices.map((processInstance) => processInstance.stop()))
932
+ await Promise.allSettled(handoffServices.map((processInstance) => processInstance.stop({retirementTimeoutMs})))
911
933
 
912
934
  // Last act of the retirement transition, before the release is recorded stopped: whatever
913
935
  // is still alive here is the residue, and the bracket pair above can now say whether it is
@@ -942,7 +964,14 @@ export default class ReleaseGroup extends EventEmitter {
942
964
  this.drainStartedAt = new Date().toISOString()
943
965
  }
944
966
 
967
+ // Refresh the live process definitions with the current retirement policy,
968
+ // but retain this generation's exact config identity for owner recovery.
969
+ // Replacing the stored generation config here would make a still-running
970
+ // singleton from the previous release fail the guardian provenance fence.
971
+ const generationConfig = this.config
972
+
945
973
  this.refreshProcessDefinitions(config)
974
+ this.config = generationConfig
946
975
  const generationIds = new Set([...this.handoffServiceIds, ...this.nonBlockingDrainIds])
947
976
  const results = await Promise.allSettled([...this.processes.entries()]
948
977
  .filter(([id]) => generationIds.has(id))
@@ -1027,6 +1056,9 @@ export default class ReleaseGroup extends EventEmitter {
1027
1056
  * @returns {{classification: ReleaseDrainStatus, currentSample: DrainLivenessSample}} The observation the decision acts on, and what the pair says.
1028
1057
  */
1029
1058
  observeRetirement({nowMs, previousSample}) {
1059
+ const drainBudgetMs = this.config.proxy.drainTimeoutMs
1060
+ const drainElapsedMs = this.drainStartedAt === undefined ? 0 : nowMs - Date.parse(this.drainStartedAt)
1061
+
1030
1062
  if (previousSample === undefined && this.connectionCount > 0) {
1031
1063
  const currentSample = this.drainNewestSample ?? {atMs: nowMs, processes: []}
1032
1064
 
@@ -1034,6 +1066,8 @@ export default class ReleaseGroup extends EventEmitter {
1034
1066
  classification: classifyDrainProgress({
1035
1067
  connectionCount: this.connectionCount,
1036
1068
  currentSample,
1069
+ drainBudgetMs,
1070
+ drainElapsedMs,
1037
1071
  minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
1038
1072
  previousSample: undefined
1039
1073
  }),
@@ -1056,6 +1090,8 @@ export default class ReleaseGroup extends EventEmitter {
1056
1090
  classification: classifyDrainProgress({
1057
1091
  connectionCount: this.connectionCount,
1058
1092
  currentSample,
1093
+ drainBudgetMs,
1094
+ drainElapsedMs,
1059
1095
  minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
1060
1096
  previousSample: priorSample
1061
1097
  }),
package/tensorbuzz.yml CHANGED
@@ -18,4 +18,6 @@ paths:
18
18
  before_script:
19
19
  - npm install
20
20
  script:
21
- - npm run all-checks
21
+ # Rollbridge's Unix control/guardian sockets must stay below the platform path limit;
22
+ # the TensorBuzz runner's workspace TMPDIR is deliberately long.
23
+ - TMPDIR=/tmp npm run all-checks
@@ -35,6 +35,8 @@ function sample(options = {}) {
35
35
  * @param {object} [options] - Classification inputs.
36
36
  * @param {number} [options.connectionCount] - Live connections owned by the generation.
37
37
  * @param {import("../src/drain-policy.js").DrainLivenessSample} [options.current] - Newest observation.
38
+ * @param {number} [options.drainBudgetMs] - Configured drain budget.
39
+ * @param {number} [options.drainElapsedMs] - Time since the drain started.
38
40
  * @param {import("../src/drain-policy.js").DrainLivenessSample | undefined} [options.previous] - Older observation.
39
41
  * @returns {import("../src/drain-policy.js").DrainClassification} Classification.
40
42
  */
@@ -42,6 +44,8 @@ function classify(options = {}) {
42
44
  return classifyDrainProgress({
43
45
  connectionCount: options.connectionCount ?? 0,
44
46
  currentSample: options.current ?? sample(),
47
+ drainBudgetMs: options.drainBudgetMs ?? 0,
48
+ drainElapsedMs: options.drainElapsedMs ?? 0,
45
49
  minSampleWindowMs: WINDOW_MS,
46
50
  previousSample: options.previous
47
51
  })
@@ -71,6 +75,30 @@ test("a generation that still owns connections is draining, never wedged", () =>
71
75
  expect(result.reason).toBe("3 live connections still owned by the generation")
72
76
  })
73
77
 
78
+ test("a live connection past the configured drain budget is reported past the deadline, not progressing", () => {
79
+ const result = classify({connectionCount: 1, current: sample({liveChildCount: 0}), drainBudgetMs: 60000, drainElapsedMs: 61000})
80
+
81
+ expect(result.status).toBe("timedOut")
82
+ expect(result.reason).toBe("drain budget 60000ms exceeded by 1000ms with 1 live connection still owned by the generation")
83
+ })
84
+
85
+ test("a live connection inside the configured drain budget is still a legitimate drain", () => {
86
+ const result = classify({connectionCount: 1, current: sample({liveChildCount: 0}), drainBudgetMs: 60000, drainElapsedMs: 30000})
87
+
88
+ expect(result.status).toBe("progressing")
89
+ expect(result.reason).toBe("1 live connection still owned by the generation")
90
+ })
91
+
92
+ test("the drain deadline names the deadline and never escalates", () => {
93
+ const currentSample = sample({liveChildCount: 0})
94
+ const classification = classify({connectionCount: 1, current: currentSample, drainBudgetMs: 60000, drainElapsedMs: 61000})
95
+ const decision = decideWedgedEscalation({classification, currentSample, minSampleWindowMs: WINDOW_MS})
96
+
97
+ expect(decision.decision).toBe("report")
98
+ expect(decision.reason).toBe("no escalation: drain is timedOut (drain budget 60000ms exceeded by 1000ms with 1 live connection still owned by the generation)")
99
+ expect(decision.targets).toEqual([])
100
+ })
101
+
74
102
  test("a process still inside its retirement budget is draining", () => {
75
103
  const result = classify({current: sample({liveChildCount: 0, retirementTimedOut: false})})
76
104
 
@@ -7,19 +7,39 @@ import os from "node:os"
7
7
  import path from "node:path"
8
8
  import {describe, expect, test} from "@velocious/testing"
9
9
  import {fileURLToPath} from "node:url"
10
- import GuardianClient from "../src/guardian-client.js"
10
+ import {createRequire} from "node:module"
11
+ import GuardianClient, {usableSocketPath} from "../src/guardian-client.js"
11
12
  import {waitForProcessExit} from "./support/process.js"
12
13
 
13
14
  describe("guardian-client", () => {
14
15
 
16
+ const commonJsRequire = createRequire(import.meta.url)
17
+ const packageVersion = /** @type {{version: string}} */ (commonJsRequire("../package.json")).version
15
18
  const legacyGuardianPath = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures", "pre-split3-process-guardian.js")
16
19
  const recoveryOwnerPath = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures", "guardian-recovery-owner.js")
17
20
 
21
+ test("guardian maps an overlong configured socket path to a stable short path", async () => {
22
+ const root = await fs.mkdtemp(path.join(os.tmpdir(), `rollbridge-${"x".repeat(120)}-`))
23
+ const configuredPath = path.join(root, "guardian.sock")
24
+ const client = new GuardianClient({socketPath: configuredPath, token: "long-path-capability"})
25
+
26
+ try {
27
+ expect(Buffer.byteLength(configuredPath)).toBeGreaterThan(107)
28
+ expect(Buffer.byteLength(client.socketPath)).toBeLessThanOrEqual(107)
29
+ expect(client.socketPath).not.toBe(configuredPath)
30
+ await client.launch()
31
+ expect(await client.capabilities()).toEqual({daemonRecovery: 1, generationReactivation: 1, guardianVersion: packageVersion})
32
+ } finally {
33
+ await cleanupGuardian({client, root})
34
+ }
35
+ })
36
+
18
37
  test("guardian bootstrap capability is absent from process argv", async () => {
19
38
  const fixture = await createGuardian()
39
+ const packageVersion = JSON.parse(await fs.readFile(path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "package.json"), "utf8")).version
20
40
 
21
41
  try {
22
- expect(await fixture.client.capabilities()).toEqual({daemonRecovery: 1, generationReactivation: 1})
42
+ expect(await fixture.client.capabilities()).toEqual({daemonRecovery: 1, generationReactivation: 1, guardianVersion: packageVersion})
23
43
  const commandLine = await fs.readFile(`/proc/${fixture.client.pid}/cmdline`, "utf8")
24
44
  const environment = await fs.readFile(`/proc/${fixture.client.pid}/environ`, "utf8")
25
45
  const status = await fs.readFile(`/proc/${fixture.client.pid}/status`, "utf8")
@@ -90,6 +110,30 @@ test("an owner mutation that outruns its lease is released and the holder report
90
110
  }
91
111
  })
92
112
 
113
+ test("a dead owner-mutation holder is reclaimed and the next mutation is admitted", async () => {
114
+ const fixture = await createGuardian()
115
+ const holder = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"])
116
+
117
+ try {
118
+ const reclaimed = fixture.client.waitForEvent("owner-mutation-reclaimed")
119
+ await fixture.client.beginOwnerMutation("restart", {ownerPid: holder.pid, waitingFor: "a process that will die"})
120
+ if (!holder.pid) throw new Error("Missing mutation holder PID")
121
+ holder.kill("SIGKILL")
122
+ await once(holder, "exit")
123
+
124
+ const successor = await fixture.client.beginOwnerMutation("deploy", {ownerPid: process.pid, waitingFor: "candidate activation"})
125
+ const event = await reclaimed
126
+
127
+ expect(typeof successor).toBe("string")
128
+ expect(event).toEqual(expect.objectContaining({code: "OWNER_MUTATION_RECLAIMED", event: "owner-mutation-reclaimed", operation: "restart", pid: holder.pid, reason: "owner mutation holder process is no longer alive"}))
129
+ expect((await fixture.client.guardianStatus()).version).toBe(packageVersion)
130
+ await fixture.client.endOwnerMutation(successor)
131
+ } finally {
132
+ if (holder.exitCode === null && holder.signalCode === null) holder.kill("SIGKILL")
133
+ await cleanupGuardian(fixture)
134
+ }
135
+ })
136
+
93
137
  test("guardian lets the owner re-register a key whose registered child is gone, and still fences a live one", async () => {
94
138
  const fixture = await createGuardian()
95
139
  const liveInstance = fixture.client.process("live-generation", definition("live-generation"))
@@ -912,7 +956,7 @@ test("replacement staging rejects owner state published after prepare", async ()
912
956
  const prepared = await candidate.prepareOwnerReplacement(authority, nextAuthority)
913
957
 
914
958
  await fixture.client.publishOwnerState({authority, snapshot: {activeReleaseId: "v2"}})
915
- await expect(candidate.stageOwnerReplacement(prepared.replacementId, {authority: nextAuthority, snapshot: {activeReleaseId: "v1"}})).rejects.toThrow(/owner state changed after prepare/i)
959
+ await expect(candidate.stageOwnerReplacement(prepared.replacementId, {authority: nextAuthority, snapshot: {activeReleaseId: "v1"}})).rejects.toThrow(/owner state changed after prepare \(prepared revision 1, current revision 2\)/i)
916
960
  await candidate.abortOwnerReplacement(prepared.replacementId)
917
961
  const fresh = await candidate.prepareOwnerReplacement(authority, nextAuthority)
918
962
 
@@ -1748,7 +1792,7 @@ test("a legacy bridge remains discoverable when its candidate disconnects after
1748
1792
  let replacement
1749
1793
 
1750
1794
  try {
1751
- const identity = {socketPath: path.join(fixture.root, "guardian-v2.sock"), token: "candidate-guardian-capability"}
1795
+ const identity = {socketPath: usableSocketPath(path.join(fixture.root, "guardian-v2.sock")), token: "candidate-guardian-capability"}
1752
1796
 
1753
1797
  upgraded = await fixture.client.upgradeLegacyGuardian({ownerState, ...identity})
1754
1798
  const prepared = await upgraded.prepareOwnerReplacement(authority, nextAuthority)
@@ -1795,7 +1839,7 @@ async function createGuardian() {
1795
1839
  async function createLegacyGuardian() {
1796
1840
  const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-legacy-guardian-"))
1797
1841
  const token = "legacy-guardian-capability"
1798
- const socketPath = path.join(root, "guardian.sock")
1842
+ const socketPath = usableSocketPath(path.join(root, "guardian.sock"))
1799
1843
  const child = spawn(process.execPath, [legacyGuardianPath, socketPath], {detached: true, stdio: ["ignore", "ignore", "ignore", "ipc"]})
1800
1844
 
1801
1845
  await new Promise((resolve, reject) => {
@@ -541,7 +541,9 @@ test("a retirement budget caps a finite graceful window instead of being outvote
541
541
  // 4,5 h and killed the in-flight builds the consumer said not to interrupt.
542
542
  expect(deadlines.map((entry) => entry.phase)).toEqual(["signal", "wait"])
543
543
  for (const entry of deadlines) {
544
- expect(entry.msFromStopStart).toBeGreaterThanOrEqual(25)
544
+ // The deadline starts at stop entry; a millisecond clock tick between this test's marker
545
+ // and the async stop call may shave 1 ms from the observed offset.
546
+ expect(entry.msFromStopStart).toBeGreaterThanOrEqual(0)
545
547
  expect(entry.msFromStopStart).toBeLessThan(1000)
546
548
  }
547
549
  expect(managed.retirementTimedOut).toBe(true)