rollbridge 0.1.67 → 0.1.69

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/daemon.js CHANGED
@@ -11,7 +11,7 @@ import {loadConfig} from "./config.js"
11
11
  import {openControlSession} from "./control-client.js"
12
12
  import {admitGenerationRelease} from "./drain-policy.js"
13
13
  import EventLog from "./event-log.js"
14
- import GuardianClient from "./guardian-client.js"
14
+ import GuardianClient, {guardianVersionFromProcess, usableSocketPath} from "./guardian-client.js"
15
15
  import {childRecordIsGone} from "./managed-process.js"
16
16
  import ReleaseGroup from "./release-group.js"
17
17
  import {clearState, isProcessAlive, liveProcesses, readState, writeState} from "./state-store.js"
@@ -25,10 +25,11 @@ const STATE_PERSIST_INTERVAL_MS = 5000
25
25
  * @typedef {{releaseId?: string, releasePath: string, revision?: string}} DeployArgs
26
26
  * @typedef {{attestation?: string, releaseId: string, releasePath: string, revision: string}} BootstrapIdentity
27
27
  * @typedef {{id: string, process: import("./managed-process.js").ManagedProcessStatus}} ProcessStatus
28
- * @typedef {{disruptive: true, mode: "legacy-first-upgrade", reason: string}} OwnerTransition
28
+ * @typedef {{disruptive: true, mode: "guardian-version-handoff" | "legacy-first-upgrade", reason: string}} OwnerTransition
29
+ * @typedef {"guardian-version-drift" | "legacy-protocol"} GuardianHandoffProtocol
29
30
  * @typedef {"candidate_ready" | "retiring_previous" | "previous_retired" | "activating_candidate" | "restoring_previous" | "retiring_failed_candidate" | "degraded_active" | "committed_pending" | "committed" | "restoring_committed"} GenerationTransitionPhase
30
31
  * @typedef {{activationError?: string, activationLifecycle?: boolean, candidateReleaseId: string, candidateReleasePath: string, candidateRevision: string, compensationError?: string, configDigest: string, degradedIncumbent?: boolean, error?: string, journalRevision?: number, phase: GenerationTransitionPhase, previousReleaseId: string | null, startedAt: string, updatedAt: string}} GenerationTransition
31
- * @typedef {{activeReleaseId: string | null, application: string, bootstrap: BootstrapIdentity | undefined, control: import("./config.js").ControlConfig, daemonPid: number, daemonRuntime: import("./daemon-runtime.js").DaemonRuntimeIdentity | undefined, generationTransition?: GenerationTransition, ownerRecovery: {configDigest: string, ready: boolean} | undefined, ownerTransition?: OwnerTransition, orphans: {id: string, pid: number, releaseId: string | null}[], proxy: {host: string, port: number | undefined, upstreamHost: string}, releaseReferences: {releaseId: string, releasePath: string}[], releases: import("./release-group.js").ReleaseStatus[], services: ProcessStatus[], singletons: ProcessStatus[]}} DaemonStatus
32
+ * @typedef {{activeReleaseId: string | null, application: string, bootstrap: BootstrapIdentity | undefined, control: import("./config.js").ControlConfig, daemonPid: number, daemonRuntime: import("./daemon-runtime.js").DaemonRuntimeIdentity | undefined, generationTransition?: GenerationTransition, guardian?: {guardianPid: number | null, ownerMutation: Record<string, JsonValue> | null, version: string | null}, ownerRecovery: {configDigest: string, ready: boolean} | undefined, ownerTransition?: OwnerTransition, orphans: {id: string, pid: number, releaseId: string | null}[], proxy: {host: string, port: number | undefined, upstreamHost: string}, releaseReferences: {releaseId: string, releasePath: string}[], releases: import("./release-group.js").ReleaseStatus[], services: ProcessStatus[], singletons: ProcessStatus[]}} DaemonStatus
32
33
  * @typedef {Omit<import("./managed-process.js").ManagedProcessStatus, "logs">} ManagedProcessStatusWithoutLogs
33
34
  * @typedef {Omit<import("./release-group.js").ReleaseStatus, "processes"> & {processes: ManagedProcessStatusWithoutLogs[]}} ReleaseStatusWithoutLogs
34
35
  * @typedef {Omit<ProcessStatus, "process"> & {process: ManagedProcessStatusWithoutLogs}} ProcessStatusWithoutLogs
@@ -37,7 +38,7 @@ const STATE_PERSIST_INTERVAL_MS = 5000
37
38
  * @typedef {{configDigest: string, format: number, guardian: {pid?: number, socketPath: string, token: string}, reconnectGraceMs: number}} OwnerRecoveryMetadata
38
39
  * @typedef {DaemonStatus & {recovery: OwnerRecoveryMetadata, serviceReleaseIds?: Record<string, string>, singletonReleaseIds?: Record<string, string>}} OwnerRecoverySnapshot
39
40
  * @typedef {{authority: JsonValue, config: import("./config.js").RollbridgeConfig, listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>, listenerSourceId?: string, recovery?: {command: JsonValue, reconnectGraceMs: number, startupTimeoutMs: number}, releaseConfigs?: Record<string, import("./config.js").RollbridgeConfig>, retiredListenerHandoff?: number, serviceReleaseIds?: Record<string, string>, singletonReleaseIds?: Record<string, string>, snapshot: OwnerRecoverySnapshot}} PrivateOwnerState
40
- * @typedef {{boundaryCrossed: boolean, incumbentControl?: Awaited<ReturnType<typeof openControlSession>>, incumbentStartTime: string, legacyGuardian: GuardianClient, legacyInventory: {key: string, provenance: string}[], legacyPrepared?: {ownerState: JsonValue, replacementId: string}, legacySnapshot: OwnerRecoverySnapshot, prepared: {ownerState: JsonValue, replacementId: string}, recoverySnapshot: OwnerRecoverySnapshot}} LegacyOwnerBridge
41
+ * @typedef {{boundaryCrossed: boolean, guardianVersion?: string, incumbentControl?: Awaited<ReturnType<typeof openControlSession>>, incumbentStartTime: string, legacyGuardian: GuardianClient, legacyInventory: {key: string, provenance: string}[], legacyPrepared?: {ownerState: JsonValue, replacementId: string}, legacySnapshot: OwnerRecoverySnapshot, prepared: {ownerState: JsonValue, replacementId: string}, protocol: GuardianHandoffProtocol, recoverySnapshot: OwnerRecoverySnapshot}} LegacyOwnerBridge
41
42
  */
42
43
 
43
44
  export default class RollbridgeDaemon {
@@ -206,7 +207,7 @@ export default class RollbridgeDaemon {
206
207
  if (!snapshot || snapshot.recovery?.guardian || !exactBootstrap) throw new Error("Retired owner reset requires an exact committed bootstrap with missing guardian identity")
207
208
  const control = await inspectControlSocket(this.config.control.path)
208
209
  if (control.alive) throw new Error(`Retired owner reset refuses responsive control owner at ${this.config.control.path}`)
209
- this.guardianIdentity = {socketPath: `${this.statePath}.guardian.sock`, token: crypto.randomBytes(32).toString("hex")}
210
+ this.guardianIdentity = {socketPath: usableSocketPath(`${this.statePath}.guardian.sock`), token: crypto.randomBytes(32).toString("hex")}
210
211
  this.guardian = new GuardianClient(this.guardianIdentity)
211
212
  await this.guardian.launch()
212
213
  this.guardianIdentity.pid = this.guardian.pid
@@ -224,10 +225,12 @@ export default class RollbridgeDaemon {
224
225
 
225
226
  if (snapshot && !recovery) throw new Error(`Owner recovery state ${this.statePath} is missing durable guardian identity; refusing to overwrite it.`)
226
227
 
227
- const guardianIdentity = recovery?.guardian || {
228
- socketPath: `${this.statePath}.guardian.sock`,
229
- token: crypto.randomBytes(32).toString("hex")
230
- }
228
+ const guardianIdentity = recovery?.guardian
229
+ ? {...recovery.guardian, socketPath: usableSocketPath(recovery.guardian.socketPath)}
230
+ : {
231
+ socketPath: usableSocketPath(`${this.statePath}.guardian.sock`),
232
+ token: crypto.randomBytes(32).toString("hex")
233
+ }
231
234
  this.guardianIdentity = guardianIdentity
232
235
  this.guardian = new GuardianClient(guardianIdentity)
233
236
  if (recovery) {
@@ -511,7 +514,7 @@ export default class RollbridgeDaemon {
511
514
  const persisted = state && typeof state === "object" && !Array.isArray(state) ? /** @type {OwnerRecoverySnapshot} */ (state) : undefined
512
515
 
513
516
  if (!persisted?.recovery?.guardian) throw new Error(`No committed durable owner transaction was found at ${this.statePath}`)
514
- this.guardianIdentity = persisted.recovery.guardian
517
+ this.guardianIdentity = {...persisted.recovery.guardian, socketPath: usableSocketPath(persisted.recovery.guardian.socketPath)}
515
518
  this.guardian = new GuardianClient(this.guardianIdentity)
516
519
  await this.guardian.connect()
517
520
  this.watchOwnerReplacementEvents()
@@ -519,9 +522,16 @@ export default class RollbridgeDaemon {
519
522
  configDigest: persisted.recovery.configDigest,
520
523
  runtime: persisted.daemonRuntime ? {...persisted.daemonRuntime} : null
521
524
  }
522
- const replacementProtocol = await this.guardian.ownerReplacementProtocol()
525
+ const guardianVersion = await this.retainedGuardianVersion()
526
+ const versionDrift = Boolean(this.runtime?.version && guardianVersion && guardianVersion !== this.runtime.version)
527
+ const replacementProtocol = versionDrift ? "legacy" : await this.guardian.ownerReplacementProtocol()
523
528
  const legacyBridge = replacementProtocol === "legacy"
524
- ? await this.prepareLegacyOwnerReplacement({persisted, persistedAuthority})
529
+ ? await this.prepareLegacyOwnerReplacement({
530
+ guardianVersion,
531
+ persisted,
532
+ persistedAuthority,
533
+ protocol: versionDrift ? "guardian-version-drift" : "legacy-protocol"
534
+ })
525
535
  : undefined
526
536
  const prepared = legacyBridge?.prepared ?? await this.guardian.prepareOwnerReplacement(persistedAuthority, this.ownerAuthority())
527
537
  let preparedStatus
@@ -767,6 +777,33 @@ export default class RollbridgeDaemon {
767
777
  }
768
778
  }
769
779
 
780
+ /**
781
+ * Reads the retained guardian's package version without deciding the replacement protocol.
782
+ *
783
+ * The version is read from the guardian's own executable rather than its capabilities
784
+ * command: a pre-replacement guardian only answers that command once a replacement
785
+ * transaction is prepared, so asking here would move the probe ahead of the transaction
786
+ * every capability-gated guardian depends on. A guardian whose version cannot be read
787
+ * stays on the protocol classification path.
788
+ * @returns {Promise<string | undefined>} Retained guardian package version.
789
+ */
790
+ async retainedGuardianVersion() {
791
+ if (!this.guardian) throw new Error("Guardian version reconciliation requires the durable process guardian")
792
+ return await guardianVersionFromProcess(this.guardian.pid)
793
+ }
794
+
795
+ /**
796
+ * Names the last-resort refusal for a guardian this daemon cannot hand off automatically.
797
+ * @param {string | undefined} guardianVersion - Retained guardian package version.
798
+ * @param {string} cause - Measured reason the automatic handoff cannot run.
799
+ * @returns {Error} Refusal naming both versions and the exact repair command.
800
+ */
801
+ guardianVersionHandoffRefusal(guardianVersion, cause) {
802
+ const configArgument = this.configPath ? ` --config ${this.configPath}` : ""
803
+
804
+ return new Error(`The persistent Rollbridge guardian is version ${guardianVersion ?? "unknown"} while this daemon runs rollbridge ${this.runtime?.version ?? "unknown"}; ${cause}. Run "rollbridge shutdown${configArgument}" and then "rollbridge ensure-daemon${configArgument}" to replace the supervisor with this Rollbridge installation.`)
805
+ }
806
+
770
807
  /** Requires the persistent guardian protocol which can restart an accepted daemon. */
771
808
  async assertGuardianDaemonRecoveryCapability() {
772
809
  if (!this.guardian) throw new Error("Owner recovery capability check requires the durable process guardian")
@@ -775,10 +812,18 @@ export default class RollbridgeDaemon {
775
812
  try {
776
813
  capabilities = await this.guardian.capabilities()
777
814
  } catch (error) {
778
- if (!(error instanceof Error) || !["Guardian capabilities requires a process key", "Unknown guardian command: capabilities"].includes(error.message)) throw error
815
+ if (!(error instanceof Error) || !isGuardianDaemonRecoveryDiagnostic(error.message)) throw error
779
816
  throw new Error("The persistent Rollbridge guardian predates daemon recovery. Perform one explicit clean Rollbridge shutdown and restart before retrying this package upgrade.", {cause: error})
780
817
  }
781
818
  if (capabilities.daemonRecovery !== 1) throw new Error("The persistent Rollbridge guardian does not support the required daemon recovery protocol")
819
+ const clientVersion = this.runtime?.version
820
+ const guardianVersion = capabilities.guardianVersion
821
+
822
+ if (!clientVersion) return
823
+ if (!guardianVersion) throw this.guardianVersionHandoffRefusal(guardianVersion, "its version cannot be attested, so no handoff can be authorised")
824
+ if (guardianVersion !== clientVersion) {
825
+ throw this.guardianVersionHandoffRefusal(guardianVersion, "this startup path has no incumbent daemon to hand guardian ownership from")
826
+ }
782
827
  }
783
828
 
784
829
  /**
@@ -899,13 +944,15 @@ export default class RollbridgeDaemon {
899
944
  }
900
945
 
901
946
  /**
902
- * Authenticates and prepares the one-time disruptive bridge for an exact legacy guardian protocol.
903
- * @param {{persisted: OwnerRecoverySnapshot, persistedAuthority: {configDigest: string, runtime: import("./daemon-runtime.js").DaemonRuntimeIdentity | null}}} options - Legacy evidence.
947
+ * Authenticates and prepares the disruptive guardian-upgrade bridge, for a guardian that predates
948
+ * the replacement protocol or whose package version drifted from this daemon's runtime.
949
+ * @param {{guardianVersion?: string, persisted: OwnerRecoverySnapshot, persistedAuthority: {configDigest: string, runtime: import("./daemon-runtime.js").DaemonRuntimeIdentity | null}, protocol: GuardianHandoffProtocol}} options - Bridge evidence and why the bridge is required.
904
950
  * @returns {Promise<LegacyOwnerBridge>} Prepared bridge.
905
951
  */
906
- async prepareLegacyOwnerReplacement({persisted, persistedAuthority}) {
952
+ async prepareLegacyOwnerReplacement({guardianVersion, persisted, persistedAuthority, protocol}) {
907
953
  const legacyProcessKey = ownerSnapshotProcessKeys(persisted, persisted.singletonReleaseIds)[0]
908
954
  const legacyGuardian = this.guardian
955
+ const versionHandoff = protocol === "guardian-version-drift"
909
956
  let legacyPrepared
910
957
 
911
958
  if (!legacyProcessKey) throw new Error("Legacy disruptive owner replacement requires an exact guardian-owned process registration in durable state")
@@ -938,12 +985,23 @@ export default class RollbridgeDaemon {
938
985
 
939
986
  try {
940
987
  if (persisted.recovery.configDigest !== this.ownerRecoveryConfigDigest()) {
988
+ if (versionHandoff) throw this.guardianVersionHandoffRefusal(guardianVersion, "the handoff keeps the incumbent config identity unchanged and this deploy changes config authority")
941
989
  throw new Error("The one-time legacy guardian bridge requires the incumbent config identity unchanged; retry the config change after the bridge establishes split-3 authority")
942
990
  }
943
- if (!this.legacyIncumbentPid) throw new Error("The authenticated legacy guardian requires an exact incumbent PID from ensure-daemon for the disruptive bridge")
991
+ if (!this.legacyIncumbentPid) {
992
+ if (versionHandoff) throw this.guardianVersionHandoffRefusal(guardianVersion, "ensure-daemon reported no incumbent daemon to hand guardian ownership from")
993
+ throw new Error("The authenticated legacy guardian requires an exact incumbent PID from ensure-daemon for the disruptive bridge")
994
+ }
944
995
  if (!this.guardianIdentity?.pid) throw new Error("The authenticated legacy guardian state is missing its exact guardian PID")
945
996
  await verifyLegacyGuardianProcess(this.guardianIdentity.pid, this.guardianIdentity.socketPath)
946
- const incumbentStartTime = await verifyLegacyDaemonProcess(this.legacyIncumbentPid, this.configPath, persisted.control.path)
997
+ let incumbentStartTime
998
+
999
+ try {
1000
+ incumbentStartTime = await verifyLegacyDaemonProcess(this.legacyIncumbentPid, this.configPath, persisted.control.path)
1001
+ } catch (error) {
1002
+ if (!versionHandoff) throw error
1003
+ throw this.guardianVersionHandoffRefusal(guardianVersion, `the incumbent daemon PID ${this.legacyIncumbentPid} is not the attested owner: ${error instanceof Error ? error.message : String(error)}`)
1004
+ }
947
1005
  if (legacyPrepared) assertLegacyPrivateOwnerState(legacyPrepared.ownerState, persisted, persistedAuthority)
948
1006
  try {
949
1007
  incumbentControl = await openControlSession(persisted.control.path)
@@ -970,7 +1028,9 @@ export default class RollbridgeDaemon {
970
1028
 
971
1029
  assertLegacyGuardianInventoryMembership(legacyInventory, /** @type {PrivateOwnerState} */ (ownerState))
972
1030
  const upgradedIdentity = /** @type {{pid?: number, socketPath: string, token: string}} */ ({
973
- socketPath: `${this.statePath}.split3-guardian.sock`,
1031
+ socketPath: usableSocketPath(versionHandoff
1032
+ ? `${this.statePath}.guardian-${this.runtime?.version}.sock`
1033
+ : `${this.statePath}.split3-guardian.sock`),
974
1034
  token: crypto.randomBytes(32).toString("hex")
975
1035
  })
976
1036
 
@@ -985,12 +1045,15 @@ export default class RollbridgeDaemon {
985
1045
  recovery: {...persisted.recovery, guardian: upgradedIdentity}
986
1046
  })
987
1047
 
988
- this.logger("legacy owner replacement bridge prepared", {
1048
+ this.logger("guardian upgrade bridge prepared", {
989
1049
  guardianPid: upgradedIdentity.pid ?? null,
1050
+ guardianVersion: guardianVersion ?? "unknown",
990
1051
  incumbentPid: this.legacyIncumbentPid,
991
- replacementId: prepared.replacementId
1052
+ protocol,
1053
+ replacementId: prepared.replacementId,
1054
+ runtimeVersion: this.runtime?.version ?? "unknown"
992
1055
  })
993
- return {boundaryCrossed: false, incumbentControl, incumbentStartTime, legacyGuardian, legacyInventory, legacyPrepared, legacySnapshot: persisted, prepared, recoverySnapshot}
1056
+ return {boundaryCrossed: false, guardianVersion, incumbentControl, incumbentStartTime, legacyGuardian, legacyInventory, legacyPrepared, legacySnapshot: persisted, prepared, protocol, recoverySnapshot}
994
1057
  } catch (upgradeError) {
995
1058
  incumbentControl?.close()
996
1059
  let cleanupError
@@ -1070,10 +1133,20 @@ export default class RollbridgeDaemon {
1070
1133
 
1071
1134
  if (staged.committed) throw new Error("Legacy guardian committed replacement before the authenticated disruptive boundary")
1072
1135
  }
1073
- this.logger("legacy owner replacement disruptive boundary", {
1136
+ const boundaryReason = bridge.protocol === "guardian-version-drift"
1137
+ ? `retained guardian ${bridge.guardianVersion ?? "unknown"} predates this daemon runtime ${this.runtime?.version ?? "unknown"}`
1138
+ : "retained guardian and daemon lacked atomic replacement protocol"
1139
+
1140
+ this.logger("guardian upgrade bridge crossed", {
1074
1141
  disruptive: true,
1142
+ guardianVersion: bridge.guardianVersion ?? "unknown",
1075
1143
  incumbentPid: this.legacyIncumbentPid,
1076
- reason: "retained guardian and daemon lacked atomic replacement protocol"
1144
+ protocol: bridge.protocol,
1145
+ reason: boundaryReason,
1146
+ // The retained guardian keeps its own children; the boundary retires only the incumbent daemon.
1147
+ retainedGuardianPid: bridge.legacyGuardian.pid ?? null,
1148
+ retainedGuardianProcesses: bridge.legacyInventory.length,
1149
+ runtimeVersion: this.runtime?.version ?? "unknown"
1077
1150
  })
1078
1151
  await this.guardian.beginLegacyOwnerClaim(
1079
1152
  bridge.prepared.replacementId,
@@ -1089,8 +1162,8 @@ export default class RollbridgeDaemon {
1089
1162
  await this.guardian.completeLegacyOwnerClaim(bridge.prepared.replacementId)
1090
1163
  this.ownerTransition = /** @type {OwnerTransition} */ ({
1091
1164
  disruptive: true,
1092
- mode: "legacy-first-upgrade",
1093
- reason: "retained guardian and daemon lacked atomic replacement protocol"
1165
+ mode: bridge.protocol === "guardian-version-drift" ? "guardian-version-handoff" : "legacy-first-upgrade",
1166
+ reason: boundaryReason
1094
1167
  })
1095
1168
  }
1096
1169
 
@@ -1406,7 +1479,18 @@ export default class RollbridgeDaemon {
1406
1479
  }
1407
1480
 
1408
1481
  if (commandName === "status") {
1409
- return this.status({includeLogs: statusIncludeLogs(data.includeLogs)})
1482
+ const status = this.status({includeLogs: statusIncludeLogs(data.includeLogs)})
1483
+ let guardian
1484
+
1485
+ try {
1486
+ const diagnostics = await this.guardian?.guardianStatus()
1487
+
1488
+ guardian = diagnostics ? {guardianPid: diagnostics.guardianPid, ownerMutation: diagnostics.ownerMutation, version: diagnostics.version} : {guardianPid: null, ownerMutation: null, version: null}
1489
+ } catch (error) {
1490
+ guardian = {error: error instanceof Error ? error.message : String(error), guardianPid: this.guardianIdentity?.pid ?? null, ownerMutation: null, version: null}
1491
+ }
1492
+
1493
+ return {...status, guardian}
1410
1494
  }
1411
1495
 
1412
1496
  if (commandName === "events") {
@@ -1956,6 +2040,8 @@ export default class RollbridgeDaemon {
1956
2040
  if (!candidate) throw new Error(`Generation transition candidate ${transition.candidateReleaseId} is not retained`)
1957
2041
  if (!previous) throw new Error(`Generation transition previous release ${transition.previousReleaseId} is not retained`)
1958
2042
  if (this.activeRelease !== previous) throw new Error(`Previous release ${previous.releaseId} is not the authoritative proxy target`)
2043
+ /** @type {Promise<void> | undefined} */
2044
+ let candidateDrain
1959
2045
 
1960
2046
  if (transition.phase === "activating_candidate") {
1961
2047
  if (!transition.error && !transition.activationError) throw new Error("Candidate activation has no recorded failure to compensate")
@@ -1987,6 +2073,15 @@ export default class RollbridgeDaemon {
1987
2073
  transition.compensationError = undefined
1988
2074
  await this.updateGenerationTransition("restoring_previous")
1989
2075
  this.logger("release generation compensation candidate retired", {releaseId: candidate.releaseId})
2076
+
2077
+ // The candidate's bounded drain starts here, before the restoration attempt. The only other
2078
+ // compensation drain runs after the success path, so when incumbent restoration throws the
2079
+ // candidate is left `draining` with a live process: measured 2026-11-17, candidate
2080
+ // `20261117063703` still held `web` pid 3201928 at 07:30Z after its compensation threw at
2081
+ // 06:42:31Z, and reached `release drained` only when a later compensation reached the
2082
+ // success path at 08:47:00Z. A retained generation holding live work is what refuses every
2083
+ // later deploy, and a failed candidate is never a restoration target.
2084
+ candidateDrain = this.drainAndPrune(candidate, candidate.config)
1990
2085
  }
1991
2086
 
1992
2087
  if (transition.phase === "restoring_previous") {
@@ -1998,10 +2093,12 @@ export default class RollbridgeDaemon {
1998
2093
 
1999
2094
  transition.compensationError = failure
2000
2095
  await this.failGenerationTransition(`Incumbent ${previous.releaseId} restoration failed: ${failure}`)
2096
+ await previous.stopProxyTraffic()
2001
2097
  this.logger("release generation compensation restoration failed", {
2002
2098
  activationError: transition.activationError,
2003
2099
  error: failure,
2004
- releaseId: previous.releaseId
2100
+ releaseId: previous.releaseId,
2101
+ traffic: "unavailable"
2005
2102
  })
2006
2103
  throw new Error(`incumbent ${previous.releaseId} restoration failed: ${failure}`, {cause: error})
2007
2104
  }
@@ -2040,7 +2137,7 @@ export default class RollbridgeDaemon {
2040
2137
  this.logger("release generation compensation checkpoint failed", {error: failure, releaseId: candidate.releaseId})
2041
2138
  throw new Error(`compensation checkpoint clear failed: ${failure}`, {cause: error})
2042
2139
  }
2043
- void this.drainAndPrune(candidate, candidate.config)
2140
+ if (candidateDrain === undefined) void this.drainAndPrune(candidate, candidate.config)
2044
2141
  this.logger("release generation compensation completed", result)
2045
2142
  return result
2046
2143
  }
@@ -3032,13 +3129,13 @@ export default class RollbridgeDaemon {
3032
3129
  await captureShutdownError(cleanupErrors, "proxy close", async () => this.proxy.close())
3033
3130
  const dependentStopResults = await Promise.allSettled([
3034
3131
  ...[...this.singletons.values()].map((processInstance) => processInstance.stop()),
3035
- ...[...this.startingReleases].map((release) => release.stop()),
3036
- ...[...this.releases.values()].map((release) => release.stop())
3132
+ ...[...this.startingReleases].map((release) => release.stop({retirementTimeoutMs: this.config.proxy.nonBlockingDrainTimeoutMs})),
3133
+ ...[...this.releases.values()].map((release) => release.stop({retirementTimeoutMs: this.config.proxy.nonBlockingDrainTimeoutMs}))
3037
3134
  ])
3038
3135
  const serviceStopResults = await Promise.allSettled([...this.services.values()].map((processInstance) => processInstance.stop()))
3039
3136
  const stopResults = [...dependentStopResults, ...serviceStopResults]
3040
3137
  const guardian = this.guardian
3041
- if (guardian) await captureShutdownError(cleanupErrors, "process guardian shutdown", () => guardian.shutdown())
3138
+ if (guardian) await captureShutdownError(cleanupErrors, "process guardian shutdown", () => guardian.shutdown({retirementTimeoutMs: this.config.proxy.nonBlockingDrainTimeoutMs}))
3042
3139
  await captureShutdownError(cleanupErrors, "proxy server close", () => this.closeServer(this.proxyServer))
3043
3140
 
3044
3141
  // Wait for any in-flight write first so it can't recreate or overwrite the final state (no
@@ -3315,6 +3412,15 @@ export function isLegacyGuardianPrepareDiagnostic(diagnostic) {
3315
3412
  diagnostic === "Unknown guardian command: prepare-owner-replacement"
3316
3413
  }
3317
3414
 
3415
+ /**
3416
+ * Reports the capability diagnostics that identify a guardian predating daemon recovery.
3417
+ * @param {string} diagnostic - Guardian error message.
3418
+ * @returns {boolean} Whether the guardian predates the daemon recovery capability.
3419
+ */
3420
+ function isGuardianDaemonRecoveryDiagnostic(diagnostic) {
3421
+ return diagnostic === "Guardian capabilities requires a process key" || diagnostic === "Unknown guardian command: capabilities"
3422
+ }
3423
+
3318
3424
  /**
3319
3425
  * @param {DaemonStatus} snapshot - Serialized owner process snapshot.
3320
3426
  * @param {Record<string, string>} [singletonReleaseIds] - Exact singleton owner releases.
@@ -3662,7 +3768,7 @@ export function releasesToPrune(releases, policy, now) {
3662
3768
  }
3663
3769
 
3664
3770
  /**
3665
- * @typedef {{alive: boolean, application?: string, activeReleaseId?: string | null, proxy?: {host: string, port: number}}} ControlSocketInspection
3771
+ * @typedef {{alive: boolean, application?: string, activeReleaseId?: string | null, guardian?: {ownerMutation: Record<string, JsonValue> | null, version: string | null}, proxy?: {host: string, port: number}}} ControlSocketInspection
3666
3772
  */
3667
3773
 
3668
3774
  /**
@@ -3737,7 +3843,7 @@ export async function inspectControlSocket(socketPath, timeoutMs = 1000) {
3737
3843
  /**
3738
3844
  * Parses a control status response line into a Rollbridge identity, if it is one.
3739
3845
  * @param {string} line - JSON response line.
3740
- * @returns {{application: string, activeReleaseId: string | null, proxy: {host: string, port: number} | undefined} | undefined} Identity, or undefined when unrecognized.
3846
+ * @returns {{application: string, activeReleaseId: string | null, guardian: {ownerMutation: Record<string, JsonValue> | null, version: string | null} | undefined, proxy: {host: string, port: number} | undefined} | undefined} Identity, or undefined when unrecognized.
3741
3847
  */
3742
3848
  function parseControlStatus(line) {
3743
3849
  /** @type {JsonValue} */
@@ -3754,5 +3860,8 @@ function parseControlStatus(line) {
3754
3860
 
3755
3861
  const proxy = "proxy" in parsed && parsed.proxy && typeof parsed.proxy === "object" && !Array.isArray(parsed.proxy) && typeof parsed.proxy.host === "string" && typeof parsed.proxy.port === "number" ? {host: parsed.proxy.host, port: parsed.proxy.port} : undefined
3756
3862
 
3757
- return {activeReleaseId: typeof parsed.activeReleaseId === "string" ? parsed.activeReleaseId : null, application: parsed.application, proxy}
3863
+ const guardianValue = "guardian" in parsed && parsed.guardian && typeof parsed.guardian === "object" && !Array.isArray(parsed.guardian) ? parsed.guardian : undefined
3864
+ const guardian = guardianValue && typeof guardianValue.version === "string" || guardianValue && guardianValue.version === null ? {ownerMutation: guardianValue.ownerMutation && typeof guardianValue.ownerMutation === "object" && !Array.isArray(guardianValue.ownerMutation) ? guardianValue.ownerMutation : null, version: guardianValue.version} : undefined
3865
+
3866
+ return {activeReleaseId: typeof parsed.activeReleaseId === "string" ? parsed.activeReleaseId : null, application: parsed.application, guardian, proxy}
3758
3867
  }
package/src/doctor.js CHANGED
@@ -26,6 +26,13 @@ export async function runEnvironmentChecks(config) {
26
26
  const socketInspection = await inspectControlSocketSafely(config.control.path)
27
27
 
28
28
  checks.push(controlSocketCheck(config, socketInspection))
29
+ if (!("error" in socketInspection) && socketInspection.guardian) {
30
+ const {guardian} = socketInspection
31
+ const lease = guardian.ownerMutation
32
+ const leaseDetail = lease ? `, owner mutation ${String(lease.operation || "unnamed operation")} held by pid ${String(lease.pid || "unknown")}` : ", no active owner-mutation lease"
33
+
34
+ checks.push({detail: `guardian Rollbridge ${guardian.version || "unknown"}${leaseDetail}`, name: "guardian diagnostics", ok: guardian.version !== null})
35
+ }
29
36
  checks.push(await controlSocketDirectoryCheck(config))
30
37
  checks.push(await proxyPortCheck(config, socketInspection))
31
38
 
@@ -1,7 +1,7 @@
1
1
  // @ts-check
2
2
 
3
3
  /**
4
- * @typedef {"progressing" | "wedged" | "unproven"} DrainProgress
4
+ * @typedef {"progressing" | "timedOut" | "wedged" | "unproven"} DrainProgress
5
5
  * @typedef {{cpuMs: number | undefined, heldProcessGroupPgid: number | undefined, id: string, liveChildCount: number | undefined, pid: number | undefined, retirementTimedOut: boolean}} DrainProcessObservation
6
6
  * @typedef {{atMs: number, processes: DrainProcessObservation[]}} DrainLivenessSample
7
7
  * @typedef {{cpuDeltaMs: number | undefined, reason: string, sampleWindowMs: number, status: DrainProgress}} DrainClassification
@@ -12,26 +12,40 @@
12
12
  */
13
13
 
14
14
  /**
15
- * Classifies one draining release group as still draining or wedged.
15
+ * Classifies one draining release group as still draining, past its drain budget, or wedged.
16
16
  *
17
17
  * A group is wedged only when its drain can no longer progress. Every inconclusive reading
18
18
  * — a live connection, a process still inside its retirement budget, an unmeasurable CPU
19
19
  * counter, or a sample window shorter than `minSampleWindowMs` — reports `progressing` or
20
20
  * `unproven`, never `wedged`: a legitimate multi-hour drain is valid and must not be
21
21
  * mislabelled as a wedge.
22
+ *
23
+ * A live connection inside `drainBudgetMs` is still a legitimate drain. Past that budget the
24
+ * connection wait has already been given up, so the group reports `timedOut` instead of
25
+ * `progressing` forever — the reading that lets one held websocket outlive the activation
26
+ * budget and stall every later deploy. `timedOut` names the deadline; it is not an escalation
27
+ * and never authorises a signal.
22
28
  * @param {object} args - Classification inputs.
23
29
  * @param {number} args.connectionCount - Live proxy connections this generation still owns.
24
30
  * @param {DrainLivenessSample} args.currentSample - Newest liveness observation.
31
+ * @param {number} args.drainBudgetMs - Configured drain budget (`proxy.drainTimeoutMs`); 0 records no budget.
32
+ * @param {number} args.drainElapsedMs - Time since this drain started, at classification time.
25
33
  * @param {number} args.minSampleWindowMs - Shortest window that can prove flat CPU.
26
34
  * @param {DrainLivenessSample | undefined} args.previousSample - Older observation, when one is in range.
27
35
  * @returns {DrainClassification} The classification and the evidence it rests on.
28
36
  */
29
- export function classifyDrainProgress({connectionCount, currentSample, minSampleWindowMs, previousSample}) {
37
+ export function classifyDrainProgress({connectionCount, currentSample, drainBudgetMs = 0, drainElapsedMs = 0, minSampleWindowMs, previousSample}) {
30
38
  const live = currentSample.processes.filter((process) => process.pid !== undefined || process.heldProcessGroupPgid !== undefined)
31
39
  const stillInsideBudget = live.filter((process) => !process.retirementTimedOut)
32
40
 
33
41
  if (connectionCount > 0) {
34
- return classify("progressing", `${connectionCount} live connection${connectionCount === 1 ? "" : "s"} still owned by the generation`, currentSample, previousSample)
42
+ const connections = `${connectionCount} live connection${connectionCount === 1 ? "" : "s"} still owned by the generation`
43
+
44
+ if (drainBudgetMs > 0 && drainElapsedMs > drainBudgetMs) {
45
+ return classify("timedOut", `drain budget ${drainBudgetMs}ms exceeded by ${drainElapsedMs - drainBudgetMs}ms with ${connections}`, currentSample, previousSample)
46
+ }
47
+
48
+ return classify("progressing", connections, currentSample, previousSample)
35
49
  }
36
50
  if (stillInsideBudget.length > 0) {
37
51
  return classify("progressing", `process ${stillInsideBudget[0].id} is still inside its retirement budget`, currentSample, previousSample)
@@ -2,17 +2,34 @@
2
2
 
3
3
  import crypto from "node:crypto"
4
4
  import {spawn} from "node:child_process"
5
+ import fs from "node:fs/promises"
5
6
  import net from "node:net"
7
+ import path from "node:path"
6
8
  import {fileURLToPath} from "node:url"
7
9
  import ManagedProcess from "./managed-process.js"
8
10
 
9
11
  const guardianPath = fileURLToPath(new URL("./process-guardian.js", import.meta.url))
12
+ const MAX_UNIX_SOCKET_PATH_BYTES = 107
13
+
14
+ /**
15
+ * Linux reserves one byte of sockaddr_un.sun_path for its terminator. Keep
16
+ * long task/workspace prefixes out of the pathname while retaining a stable,
17
+ * collision-resistant name for the configured path.
18
+ * @param {string} socketPath - Configured Unix socket pathname.
19
+ * @returns {string} Path that can be passed to net.createConnection/listen.
20
+ */
21
+ export function usableSocketPath(socketPath) {
22
+ if (Buffer.byteLength(socketPath) <= MAX_UNIX_SOCKET_PATH_BYTES) return socketPath
23
+
24
+ const digest = crypto.createHash("sha256").update(socketPath).digest("hex").slice(0, 32)
25
+ return path.join("/tmp", `rollbridge-${digest}.sock`)
26
+ }
10
27
 
11
28
  export default class GuardianClient {
12
29
  /** @param {{pid?: number, socketPath: string, token: string}} identity - Durable guardian identity. */
13
30
  constructor({pid, socketPath, token}) {
14
31
  this.pid = pid
15
- this.socketPath = socketPath
32
+ this.socketPath = usableSocketPath(socketPath)
16
33
  this.token = token
17
34
  this.socket = /** @type {net.Socket | undefined} */ (undefined)
18
35
  this.buffer = ""
@@ -150,14 +167,17 @@ export default class GuardianClient {
150
167
  return await response
151
168
  }
152
169
 
153
- /** Stops the guardian after every owned process has stopped. */
154
- async shutdown() {
170
+ /**
171
+ * Stops the guardian after every owned process has stopped or reached its retirement budget.
172
+ * @param {{retirementTimeoutMs?: number}} [options] - Optional non-blocking retirement budget.
173
+ */
174
+ async shutdown(options = {}) {
155
175
  if (this.pending.size > 0) {
156
176
  await new Promise((resolve) => {
157
177
  this.idleWaiters.push(() => { resolve(undefined) })
158
178
  })
159
179
  }
160
- await this.request({command: "shutdown"})
180
+ await this.request({command: "shutdown", options})
161
181
  const socket = this.socket
162
182
 
163
183
  if (!socket || socket.destroyed) throw new Error("Process guardian disconnected before shutdown acknowledgement")
@@ -186,9 +206,11 @@ export default class GuardianClient {
186
206
  await this.request({command: "owner-ready", ownerPid: process.pid})
187
207
  }
188
208
 
189
- /** @returns {Promise<{daemonRecovery: number, generationReactivation?: number}>} Guardian protocol capabilities. */
209
+ /** @returns {Promise<{daemonRecovery: number, generationReactivation?: number, guardianVersion?: string}>} Guardian protocol capabilities. */
190
210
  async capabilities() {
191
- const capabilities = /** @type {{daemonRecovery: number, generationReactivation?: number}} */ (await this.request({command: "capabilities"}))
211
+ const capabilities = /** @type {{daemonRecovery: number, generationReactivation?: number, guardianVersion?: string}} */ (await this.request({command: "capabilities"}))
212
+
213
+ if (!capabilities.guardianVersion) capabilities.guardianVersion = await guardianVersionFromProcess(this.pid)
192
214
 
193
215
  this.generationReactivation = capabilities.generationReactivation === 1
194
216
  return capabilities
@@ -347,6 +369,11 @@ export default class GuardianClient {
347
369
  await this.request({command: "report-owner-mutation-wait", mutationId, waitingFor})
348
370
  }
349
371
 
372
+ /** @returns {Promise<{guardianPid: number, ownerMutation: {elapsedMs: number, leaseMs: number, mutationId: string, operation: string, pid: number | null, startedAt: string, waitingFor: string} | null, version: string}>} Guardian diagnostics. */
373
+ async guardianStatus() {
374
+ return /** @type {{guardianPid: number, ownerMutation: {elapsedMs: number, leaseMs: number, mutationId: string, operation: string, pid: number | null, startedAt: string, waitingFor: string} | null, version: string}} */ (await this.request({command: "guardian-status"}))
375
+ }
376
+
350
377
  /** @returns {Promise<{committedReplacementId: string | null, ownerClaimed: boolean, retirementFailed?: boolean, retirementPending?: boolean, retirementReady?: boolean}>} Transaction status. */
351
378
  async replacementStatus() {
352
379
  return /** @type {{committedReplacementId: string | null, ownerClaimed: boolean, retirementFailed?: boolean, retirementPending?: boolean, retirementReady?: boolean}} */ (await this.request({command: "replacement-status"}))
@@ -450,6 +477,27 @@ export default class GuardianClient {
450
477
  }
451
478
  }
452
479
 
480
+ /**
481
+ * Reads the package version from a guardian's own executable path. Used when its
482
+ * capabilities response predates guardian version reporting, and by owner
483
+ * replacement, which must not send a capabilities command before a replacement
484
+ * transaction is prepared. An unreadable or non-Rollbridge process remains unknown.
485
+ * @param {number | undefined} pid - Existing guardian PID.
486
+ * @returns {Promise<string | undefined>} Guardian package version.
487
+ */
488
+ export async function guardianVersionFromProcess(pid) {
489
+ if (!pid) return undefined
490
+
491
+ try {
492
+ const commandLine = (await fs.readFile(`/proc/${pid}/cmdline`, "utf8")).split("\0").find((argument) => argument.endsWith("/src/process-guardian.js"))
493
+ if (!commandLine) return undefined
494
+ const metadata = JSON.parse(await fs.readFile(path.resolve(path.dirname(commandLine), "..", "package.json"), "utf8"))
495
+ return metadata && typeof metadata.version === "string" ? metadata.version : undefined
496
+ } catch {
497
+ return undefined
498
+ }
499
+ }
500
+
453
501
  class GuardianProcess extends ManagedProcess {
454
502
  /** @param {{client: GuardianClient, definition: ConstructorParameters<typeof ManagedProcess>[0], key: string}} args - Remote process args. */
455
503
  constructor({client, definition, key}) {
@@ -22,7 +22,7 @@ const ESCALATION_KILL_CONFIRM_MS = 5000
22
22
  * @typedef {import("node:child_process").ChildProcess["signalCode"]} ProcessExitSignal
23
23
  * @typedef {{at: string, line: string, stream: "stdout" | "stderr"}} ManagedProcessLog
24
24
  * @typedef {import("./config.js").StopTimeoutMs} StopTimeoutMs
25
- * @typedef {{command: string, cwd: string | undefined, env: Record<string, string | undefined>, lifecycle: import("./config.js").LifecycleConfig, logger: (message: string, data?: Record<string, import("./json.js").JsonValue>) => void, memory: import("./config.js").MemoryConfig | undefined, outputLines: number, restart: import("./config.js").RestartConfig, restartDelayMs: number, shouldRestart: () => boolean, stopSignal: string, stopTimeoutMs: StopTimeoutMs}} ManagedProcessDefinition
25
+ * @typedef {{command: string, cwd: string | undefined, env: Record<string, string | undefined>, lifecycle: import("./config.js").LifecycleConfig, logger: (message: string, data?: Record<string, import("./json.js").JsonValue>) => void, memory: import("./config.js").MemoryConfig | undefined, outputLines: number, restart: import("./config.js").RestartConfig, restartDelayMs: number, shouldRestart: () => boolean, shouldStop?: () => boolean, stopSignal: string, stopTimeoutMs: StopTimeoutMs}} ManagedProcessDefinition
26
26
  * @typedef {{liveMemberCount: number | undefined, pgid: number}} HeldProcessGroup
27
27
  * @typedef {{children: import("./process-memory.js").ProcessGroupMember[], command: string, cwd: string | undefined, exitCode: number | null | undefined, exitSignal: ProcessExitSignal | undefined, heldProcessGroup: HeldProcessGroup | undefined, id: string, lastMemoryRestartAt: string | undefined, lastStartReason: ManagedProcessStartReason | undefined, lifecycleRole?: LifecycleRole, logs: ManagedProcessLog[], memoryRestarts: number, pid: number | undefined, restarts: number, retirementTimedOut: boolean, rssBytes: number | undefined, startedAt: string | undefined, state: ManagedProcessState, uptimeMs: number | undefined}} ManagedProcessStatus
28
28
  */
@@ -41,10 +41,11 @@ export default class ManagedProcess extends EventEmitter {
41
41
  * @param {import("./config.js").RestartConfig} [args.restart] - Restart policy (defaults to unlimited restarts with a constant delay).
42
42
  * @param {number} args.restartDelayMs - Restart delay.
43
43
  * @param {() => boolean} args.shouldRestart - Restart policy callback.
44
+ * @param {() => boolean} [args.shouldStop] - Whether an unexpected exit is terminal because its dependency retired.
44
45
  * @param {string} [args.stopSignal] - Signal sent to gracefully stop the process (default "SIGTERM").
45
46
  * @param {StopTimeoutMs} args.stopTimeoutMs - Stop timeout.
46
47
  */
47
- constructor({command, cwd, env, id, lifecycle = {drainTimeoutMs: 0}, logger, memory, outputLines, restart = {backoffFactor: 1, maxDelayMs: 0, maxRestarts: undefined, windowMs: 0}, restartDelayMs, shouldRestart, stopSignal = "SIGTERM", stopTimeoutMs}) {
48
+ constructor({command, cwd, env, id, lifecycle = {drainTimeoutMs: 0}, logger, memory, outputLines, restart = {backoffFactor: 1, maxDelayMs: 0, maxRestarts: undefined, windowMs: 0}, restartDelayMs, shouldRestart, shouldStop = () => false, stopSignal = "SIGTERM", stopTimeoutMs}) {
48
49
  super()
49
50
 
50
51
  this.command = command
@@ -58,6 +59,7 @@ export default class ManagedProcess extends EventEmitter {
58
59
  this.restart = restart
59
60
  this.restartDelayMs = restartDelayMs
60
61
  this.shouldRestart = shouldRestart
62
+ this.shouldStop = shouldStop
61
63
  this.stopSignal = stopSignal
62
64
  this.stopTimeoutMs = stopTimeoutMs
63
65
  this.state = /** @type {ManagedProcessState} */ ("stopped")
@@ -207,6 +209,7 @@ export default class ManagedProcess extends EventEmitter {
207
209
  this.restart = definition.restart
208
210
  this.restartDelayMs = definition.restartDelayMs
209
211
  this.shouldRestart = definition.shouldRestart
212
+ this.shouldStop = definition.shouldStop ?? (() => false)
210
213
  this.stopSignal = definition.stopSignal
211
214
  this.stopTimeoutMs = definition.stopTimeoutMs
212
215
  }
@@ -265,12 +268,13 @@ export default class ManagedProcess extends EventEmitter {
265
268
  this.rssBytes = undefined
266
269
  this.children = []
267
270
  this.clearMemoryMonitor()
268
- this.state = wasIntentional ? "stopped" : "failed"
271
+ const terminalExit = !wasIntentional && this.shouldStop()
272
+ this.state = wasIntentional || terminalExit ? "stopped" : "failed"
269
273
 
270
274
  this.logger("process exited", {code, id: this.id, signal})
271
275
  this.emit("exit", {code, signal})
272
276
 
273
- if (!wasIntentional && this.shouldRestart()) {
277
+ if (!wasIntentional && !terminalExit && this.shouldRestart()) {
274
278
  this.scheduleRestart()
275
279
  }
276
280
  }
@@ -507,6 +511,7 @@ export default class ManagedProcess extends EventEmitter {
507
511
  async performStop(options) {
508
512
  const pgid = this.child?.pid ?? this.pid
509
513
  const exitPromise = this.exitPromise
514
+ const retirementDeadline = options.retirementTimeoutMs === undefined ? undefined : Date.now() + options.retirementTimeoutMs
510
515
  await this.quiesce()
511
516
 
512
517
  if (!pgid) {
@@ -528,7 +533,6 @@ export default class ManagedProcess extends EventEmitter {
528
533
  // 3. Stop whatever is still running, then SIGKILL if it outlasts the graceful window.
529
534
  if (this.processGroupExists(pgid)) {
530
535
  const timeoutMs = options.timeoutMs ?? this.stopTimeoutMs
531
- const retirementDeadline = options.retirementTimeoutMs === undefined ? undefined : Date.now() + options.retirementTimeoutMs
532
536
  const gracefulDeadline = timeoutMs === "indefinite" ? undefined : Date.now() + timeoutMs
533
537
  // A retirement budget is the caller's decision that this process must not be waited for and
534
538
  // must not be killed, so it caps the graceful window rather than only standing in for a
@@ -550,6 +554,11 @@ export default class ManagedProcess extends EventEmitter {
550
554
  if (!exited && retirementBound) {
551
555
  this.retirementTimedOut = true
552
556
  this.markGroupOutlivedStop(pgid)
557
+ // The tracked process has completed its supervised stop even though its process group is
558
+ // still draining. Keep the residue in heldProcessGroup rather than leaving the process
559
+ // permanently quiesced, which would make a drained generation look active to restore
560
+ // admission and keep the owner state from describing the actual stopped generation.
561
+ this.state = "stopped"
553
562
  this.logger("process retirement wait timed out; leaving process alive", {id: this.id, pid: pgid, timeoutMs: options.retirementTimeoutMs})
554
563
  return
555
564
  }