rollbridge 0.1.28 → 0.1.30

package/src/release-group.js

@@ -10,8 +10,8 @@ import {waitForHealth} from "./health.js"
  * @typedef {import("./json.js").JsonValue} JsonValue
  * @typedef {"starting" | "active" | "draining" | "stopped" | "failed"} ReleaseState
  * @typedef {{http: number, websocket: number}} ReleaseConnections
- * @typedef {{activatedAt: string | undefined, connectionCount: number, connections: ReleaseConnections, drainStartedAt: string | undefined, ports: Record<string, number>, processes: import("./managed-process.js").ManagedProcessStatus[], releaseId: string, releasePath: string, revision: string, state: ReleaseState, stoppedAt: string | undefined}} ReleaseStatus
- * @typedef {{count?: number, index?: number, instanceId?: string, shouldRestart?: () => boolean}} BuildProcessOptions
+ * @typedef {{activatedAt: string | undefined, connectionCount: number, connections: ReleaseConnections, drainStartedAt: string | undefined, ports: Record<string, number>, processes: import("./managed-process.js").ManagedProcessStatus[], releaseId: string, releasePath: string, retirementError: string | undefined, revision: string, state: ReleaseState, stoppedAt: string | undefined}} ReleaseStatus
+ * @typedef {{count?: number, guardianKey?: string, index?: number, instanceId?: string, shouldRestart?: () => boolean}} BuildProcessOptions
  */
 
 /**
@@ -36,13 +36,15 @@ export default class ReleaseGroup extends EventEmitter {
    * @param {object} args - Options.
    * @param {import("./config.js").RollbridgeConfig} args.config - Rollbridge config.
    * @param {(message: string, data?: Record<string, JsonValue>) => void} args.logger - Logger.
+   * @param {Set<number>} [args.portReservations] - Ports owned by live generations and daemon services.
    * @param {string} args.releaseId - Release id.
    * @param {string} args.releasePath - Release path.
    * @param {string | undefined} args.revision - Revision.
    * @param {Record<string, number>} [args.servicePorts] - Ports already owned by daemon-wide services.
    * @param {() => boolean} [args.shouldStart] - Whether bootstrap may create another process.
+   * @param {(key: string, definition: ConstructorParameters<typeof ManagedProcess>[0]) => ManagedProcess} [args.processFactory] - Durable process factory.
    */
-  constructor({config, logger, releaseId, releasePath, revision, servicePorts = {}, shouldStart = () => true}) {
+  constructor({config, logger, portReservations = new Set(), processFactory, releaseId, releasePath, revision, servicePorts = {}, shouldStart = () => true}) {
     super()
 
     this.config = config
@@ -53,16 +55,23 @@ export default class ReleaseGroup extends EventEmitter {
     this.state = /** @type {ReleaseState} */ ("starting")
     this.connectionCount = 0
     this.connections = /** @type {ReleaseConnections} */ ({http: 0, websocket: 0})
+    this.transferredConnections = /** @type {ReleaseConnections} */ ({http: 0, websocket: 0})
+    this.ownerHandoffPaused = false
     this.processes = /** @type {Map<string, ManagedProcess>} */ (new Map())
     this.handoffServiceIds = /** @type {Set<string>} */ (new Set())
     this.nonBlockingDrainIds = /** @type {Set<string>} */ (new Set())
     this.ports = /** @type {Record<string, number>} */ ({})
+    this.portReservations = portReservations
+    this.ownedPortReservations = /** @type {Set<number>} */ (new Set())
     this.servicePorts = servicePorts
     this.shouldStart = shouldStart
+    this.processFactory = processFactory
     this.portsAllocated = false
     this.drainStartedAt = /** @type {string | undefined} */ (undefined)
     this.activatedAt = /** @type {string | undefined} */ (undefined)
     this.stoppedAt = /** @type {string | undefined} */ (undefined)
+    this.retirementError = /** @type {string | undefined} */ (undefined)
+    this.preserveConfigOnRetirement = false
   }
 
   /** @returns {Promise<void>} Starts release-owned processes and health checks the proxied process. */
@@ -104,6 +113,70 @@ export default class ReleaseGroup extends EventEmitter {
     }
   }
 
+  /**
+   * Reconstructs this release around processes still owned by the durable guardian.
+   * @param {ReleaseStatus} snapshot - Persisted release snapshot.
+   */
+  async restore(snapshot) {
+    if (!snapshot || snapshot.releaseId !== this.releaseId || snapshot.releasePath !== this.releasePath || snapshot.revision !== this.revision) {
+      throw new Error(`Persisted release identity mismatch for ${this.releaseId}`)
+    }
+    if (!snapshot.ports || !Array.isArray(snapshot.processes)) throw new Error(`Persisted release ${this.releaseId} is missing ports or processes`)
+
+    const expectedProcessIds = this.config.processes
+      .filter((processConfig) => processConfig.policy !== "singleton" && (processConfig.policy !== "service" || processConfig.deployStrategy === "handoff"))
+      .flatMap((processConfig) => Array.from({length: processConfig.replicas}, (_, index) => replicaInstanceId(processConfig, index)))
+      .sort()
+    const persistedProcessIds = snapshot.processes.map((processStatus) => processStatus.id).sort()
+
+    if (JSON.stringify(persistedProcessIds) !== JSON.stringify(expectedProcessIds)) {
+      throw new Error(`Persisted release ${this.releaseId} process set does not match the configured generation`)
+    }
+    for (const processConfig of this.config.processes) {
+      if (processConfig.port && typeof snapshot.ports[processConfig.id] !== "number") throw new Error(`Persisted release ${this.releaseId} is missing port ${processConfig.id}`)
+    }
+
+    const generationPorts = this.config.processes
+      .filter((processConfig) => processConfig.port && (processConfig.policy !== "service" || processConfig.deployStrategy === "handoff"))
+      .map((processConfig) => snapshot.ports[processConfig.id])
+    const distinctGenerationPorts = new Set(generationPorts)
+
+    if (distinctGenerationPorts.size !== generationPorts.length) throw new Error(`Persisted release ${this.releaseId} reuses a port within one live generation`)
+    for (const port of distinctGenerationPorts) {
+      if (this.portReservations.has(port)) throw new Error(`Persisted release ${this.releaseId} port ${port} is already reserved by another live generation`)
+    }
+    for (const port of distinctGenerationPorts) {
+      this.portReservations.add(port)
+      this.ownedPortReservations.add(port)
+    }
+
+    this.ports = {...snapshot.ports}
+    this.portsAllocated = true
+    this.state = snapshot.state
+    this.activatedAt = snapshot.activatedAt
+    this.drainStartedAt = snapshot.drainStartedAt
+    this.retirementError = snapshot.retirementError
+    this.stoppedAt = snapshot.stoppedAt
+    this.setTransferredConnections(snapshot.connections)
+
+    for (const processStatus of snapshot.processes) {
+      const baseId = processStatus.id.replace(/#\d+$/, "")
+      const processConfig = this.config.processes.find((candidate) => candidate.id === baseId)
+
+      if (!processConfig || processConfig.policy === "singleton" || (processConfig.policy === "service" && processConfig.deployStrategy !== "handoff")) {
+        throw new Error(`Persisted release ${this.releaseId} contains unknown process ${processStatus.id}`)
+      }
+      const replicaMatch = processStatus.id.match(/#(\d+)$/)
+      const index = replicaMatch ? Number(replicaMatch[1]) : 0
+      const processInstance = this.buildProcess(processConfig, {count: processConfig.replicas, index, instanceId: processStatus.id})
+
+      this.processes.set(processStatus.id, processInstance)
+      if (processConfig.policy === "service" && processConfig.deployStrategy === "handoff") this.handoffServiceIds.add(processStatus.id)
+      if (processConfig.nonBlockingDrain) this.nonBlockingDrainIds.add(processStatus.id)
+      if ("recover" in processInstance && typeof processInstance.recover === "function") await processInstance.recover()
+    }
+  }
+
   /**
    * @param {string} id - Process id.
    * @returns {ManagedProcess | undefined} This release's managed process with the given id, if present.
@@ -184,26 +257,43 @@ export default class ReleaseGroup extends EventEmitter {
   async allocatePorts() {
     if (this.portsAllocated) return
 
-    const usedPorts = /** @type {Set<number>} */ (new Set())
-
     for (const processConfig of this.config.processes) {
       if (!processConfig.port) continue
       if (processConfig.policy === "service" && processConfig.deployStrategy !== "handoff" && this.servicePorts[processConfig.id] !== undefined) {
         this.ports[processConfig.id] = this.servicePorts[processConfig.id]
-        usedPorts.add(this.servicePorts[processConfig.id])
         continue
       }
 
-      this.ports[processConfig.id] = await findAvailablePort({
+      const port = await findAvailablePort({
         host: this.config.proxy.upstreamHost,
         range: processConfig.port,
-        usedPorts
+        usedPorts: this.portReservations
       })
+
+      this.ports[processConfig.id] = port
+      this.ownedPortReservations.add(port)
     }
 
     this.portsAllocated = true
   }
 
+  /**
+   * Transfers a newly allocated daemon-wide service port out of this generation's cleanup scope.
+   * @param {string} processId - Daemon-wide service process id.
+   * @returns {void}
+   */
+  transferPortReservation(processId) {
+    const port = this.ports[processId]
+
+    if (port !== undefined) this.ownedPortReservations.delete(port)
+  }
+
+  /** @returns {void} Releases ports only after this generation has truly stopped. */
+  releasePortReservations() {
+    for (const port of this.ownedPortReservations) this.portReservations.delete(port)
+    this.ownedPortReservations.clear()
+  }
+
   /**
    * Builds a managed process from config.
    * @param {import("./config.js").ProcessConfig} processConfig - Process config.
@@ -221,7 +311,7 @@ export default class ReleaseGroup extends EventEmitter {
       ...renderedEnv
     }
 
-    return new ManagedProcess({
+    const definition = /** @type {ConstructorParameters<typeof ManagedProcess>[0]} */ ({
       command: renderTemplate(processConfig.command, context),
       cwd: processConfig.cwd ? renderTemplate(processConfig.cwd, context) : this.releasePath,
       env: processEnv,
@@ -236,6 +326,10 @@ export default class ReleaseGroup extends EventEmitter {
       stopSignal: processConfig.stopSignal,
       stopTimeoutMs: processConfig.gracefulStopMs
     })
+
+    return this.processFactory
+      ? this.processFactory(options.guardianKey || `release:${this.releaseId}:${instanceId}`, definition)
+      : new ManagedProcess(definition)
   }
 
   /**
@@ -356,6 +450,48 @@ export default class ReleaseGroup extends EventEmitter {
     }
   }
 
+  /**
+   * Reconciles connections still owned by a prior daemon listener.
+   * @param {ReleaseConnections} connections - Exact incumbent listener counts.
+   */
+  setTransferredConnections(connections) {
+    const http = connections?.http
+    const websocket = connections?.websocket
+
+    if (!Number.isSafeInteger(http) || http < 0 || !Number.isSafeInteger(websocket) || websocket < 0) {
+      throw new Error(`Persisted release ${this.releaseId} has invalid listener connection counts`)
+    }
+    const previous = this.transferredConnections
+
+    this.connectionCount += http + websocket - previous.http - previous.websocket
+    this.connections.http += http - previous.http
+    this.connections.websocket += websocket - previous.websocket
+    this.transferredConnections = {http, websocket}
+    if (this.connectionCount === 0) this.emit("drained")
+  }
+
+  /** @returns {boolean} Whether a prior daemon still owns live connections for this release. */
+  hasTransferredConnections() {
+    return this.transferredConnections.http + this.transferredConnections.websocket > 0
+  }
+
+  /** Pauses only daemon-local connection-dependent retirement at owner handoff. */
+  pauseDrainForOwnerHandoff() {
+    if (this.state !== "draining") return
+    this.ownerHandoffPaused = true
+    this.emit("ownerHandoff")
+  }
+
+  /** @returns {boolean} Whether retirement completion belongs to the prepared successor. */
+  isDrainPausedForOwnerHandoff() {
+    return this.ownerHandoffPaused
+  }
+
+  /** Resumes a drain after its prepared owner handoff aborts. */
+  resumeDrainAfterOwnerHandoff() {
+    this.ownerHandoffPaused = false
+  }
+
   /**
    * Starts draining and stops once existing connections close or timeout.
    * @param {number} timeoutMs - Drain timeout.
@@ -365,9 +501,7 @@ export default class ReleaseGroup extends EventEmitter {
   async drainAndStop(timeoutMs, config = this.config) {
     if (this.state === "stopped") return
 
-    this.state = "draining"
-    this.drainStartedAt = new Date().toISOString()
-    this.refreshProcessDefinitions(config)
+    if (this.state !== "draining") await this.beginRetirement(config)
 
     // Stop nonBlockingDrain processes (e.g. job workers) immediately and in the background, so
     // their lifecycle drain runs as soon as the release is retired — in parallel with the
@@ -379,19 +513,58 @@ export default class ReleaseGroup extends EventEmitter {
 
     if (this.connectionCount > 0) {
       await new Promise((resolve) => {
-        const timer = setTimeout(resolve, timeoutMs)
-        this.once("drained", () => {
+        let timer = /** @type {ReturnType<typeof setTimeout> | undefined} */ (undefined)
+        const completed = () => {
           clearTimeout(timer)
+          this.off("drained", completed)
+          this.off("ownerHandoff", completed)
           resolve(undefined)
-        })
+        }
+
+        this.once("drained", completed)
+        this.once("ownerHandoff", completed)
+        timer = setTimeout(completed, timeoutMs)
       })
     }
 
+    if (this.ownerHandoffPaused) {
+      void Promise.allSettled(nonBlockingStops)
+      return
+    }
+
     await Promise.allSettled(connectionDependent.map((processInstance) => processInstance.stop()))
     await Promise.allSettled(nonBlockingStops)
     await Promise.allSettled(handoffServices.map((processInstance) => processInstance.stop()))
     this.state = "stopped"
     this.stoppedAt = new Date().toISOString()
+    this.releasePortReservations()
+  }
+
+  /**
+   * Marks the generation retired and quiesces its jobs-main and non-blocking workers as one unit.
+   * @param {import("./config.js").RollbridgeConfig} [config] - Refreshed retirement config.
+   * @returns {Promise<void>} Resolves when retirement quiescence succeeds.
+   */
+  async beginRetirement(config = this.config) {
+    if (this.state === "draining") {
+      if (this.retirementError) throw new Error(this.retirementError)
+      return
+    }
+
+    this.state = "draining"
+    this.drainStartedAt = new Date().toISOString()
+    this.refreshProcessDefinitions(config)
+    const generationIds = new Set([...this.handoffServiceIds, ...this.nonBlockingDrainIds])
+    const results = await Promise.allSettled([...this.processes.entries()]
+      .filter(([id]) => generationIds.has(id))
+      .map(([, processInstance]) => processInstance.quiesceStrict()))
+    const errors = results.filter((result) => result.status === "rejected").map((result) => result.reason)
+
+    if (errors.length > 0) {
+      const failure = new AggregateError(errors, `Release ${this.releaseId} retirement quiescence failed`)
+      this.retirementError = `${failure.message}: ${errors.map((error) => error instanceof Error ? error.message : String(error)).join("; ")}`
+      throw failure
+    }
   }
 
   /** @returns {Promise<void>} Stops all release-owned processes. */
@@ -401,6 +574,7 @@ export default class ReleaseGroup extends EventEmitter {
     await Promise.allSettled(stopTasks)
     this.state = "stopped"
     this.stoppedAt = new Date().toISOString()
+    this.releasePortReservations()
   }
 
   /** @returns {Promise<void>} Quiesces every release process without waiting for its drain. */
@@ -419,6 +593,7 @@ export default class ReleaseGroup extends EventEmitter {
       processes: [...this.processes.values()].map((processInstance) => processInstance.status()),
       releaseId: this.releaseId,
       releasePath: this.releasePath,
+      retirementError: this.retirementError,
       revision: this.revision,
       state: this.state,
       stoppedAt: this.stoppedAt

package/src/state-store.js

@@ -21,7 +21,7 @@ export async function writeState(path, state) {
 
   const tempPath = `${path}.${process.pid}.${tempCounter}.tmp`
 
-  await fs.writeFile(tempPath, `${JSON.stringify(state, null, 2)}\n`)
+  await fs.writeFile(tempPath, `${JSON.stringify(state, null, 2)}\n`, {mode: 0o600})
   await fs.rename(tempPath, path)
 }
 

package/test/config-validation.test.js

@@ -571,6 +571,28 @@ test("validateConfig leaves statePath unset by default, accepts a string, and re
   assert.ok(validateStatePath(123).issues.some((issue) => issue.message === "statePath must be a string"))
 })
 
+test("ownerRecovery requires durable state and a non-negative integer reconnection grace", () => {
+  const raw = {
+    application: "demo",
+    control: {path: "/tmp/demo.sock"},
+    ownerRecovery: {reconnectGraceMs: 45000},
+    processes: [{command: "run web", id: "web", policy: "proxied", port: {from: 18000, to: 18099}}],
+    proxy: {host: "127.0.0.1", port: 8182}
+  }
+  const missingState = validateConfig(raw)
+
+  assert.ok(missingState.issues.some((issue) => issue.message === "ownerRecovery requires statePath"))
+
+  const valid = validateConfig({...raw, statePath: "/var/lib/rollbridge/demo.state.json"})
+
+  assert.deepEqual(valid.issues, [])
+  assert.deepEqual(valid.config.ownerRecovery, {reconnectGraceMs: 45000})
+
+  const invalidGrace = validateConfig({...raw, ownerRecovery: {reconnectGraceMs: -1}, statePath: "/var/lib/rollbridge/demo.state.json"})
+
+  assert.ok(invalidGrace.issues.some((issue) => issue.message === "ownerRecovery.reconnectGraceMs must be a non-negative integer"))
+})
+
 test("normalizeConfig throws an aggregated error listing every issue", () => {
   assert.throws(
     () => normalizeConfig({

package/test/fixtures/pre-split3-daemon-runner.js

@@ -0,0 +1,30 @@
+// @ts-check
+
+import {loadConfig} from "../../src/config.js"
+import {currentPackageIdentity} from "../../src/daemon-runtime.js"
+import RollbridgeDaemon from "./pre-split3-daemon.js"
+
+const [command, configFlag, configPath] = process.argv.slice(2)
+
+if (command !== "daemon" || configFlag !== "--config" || !configPath) throw new Error("pre-split3-daemon-runner requires daemon --config <path>")
+
+const daemon = new RollbridgeDaemon({
+  config: await loadConfig(configPath),
+  configPath,
+  runtime: await currentPackageIdentity()
+})
+
+await daemon.start()
+
+const shutdown = async () => {
+  try {
+    await daemon.shutdown()
+    process.exit(0)
+  } catch (error) {
+    console.error(error)
+    process.exit(1)
+  }
+}
+
+process.once("SIGINT", () => { void shutdown() })
+process.once("SIGTERM", () => { void shutdown() })