rollbridge 0.1.39 → 0.1.41

package/src/release-group.js

@@ -141,14 +141,17 @@ export default class ReleaseGroup extends EventEmitter {
       .filter((processConfig) => processConfig.port && (processConfig.policy !== "service" || processConfig.deployStrategy === "handoff"))
       .map((processConfig) => snapshot.ports[processConfig.id])
     const distinctGenerationPorts = new Set(generationPorts)
+    const ownsGenerationPorts = snapshot.state === "starting" || snapshot.state === "active" || snapshot.state === "draining"
 
     if (distinctGenerationPorts.size !== generationPorts.length) throw new Error(`Persisted release ${this.releaseId} reuses a port within one live generation`)
-    for (const port of distinctGenerationPorts) {
-      if (this.portReservations.has(port)) throw new Error(`Persisted release ${this.releaseId} port ${port} is already reserved by another live generation`)
-    }
-    for (const port of distinctGenerationPorts) {
-      this.portReservations.add(port)
-      this.ownedPortReservations.add(port)
+    if (ownsGenerationPorts) {
+      for (const port of distinctGenerationPorts) {
+        if (this.portReservations.has(port)) throw new Error(`Persisted release ${this.releaseId} port ${port} is already reserved by another live generation`)
+      }
+      for (const port of distinctGenerationPorts) {
+        this.portReservations.add(port)
+        this.ownedPortReservations.add(port)
+      }
     }
 
     this.ports = {...snapshot.ports}
@@ -276,11 +279,34 @@ export default class ReleaseGroup extends EventEmitter {
     await instance.process.activateStrict()
   }
 
+  /** Restores the retained generation coordinator to its active role in place. */
+  async reactivateGeneration() {
+    if (this.state !== "draining" && this.state !== "active") throw new Error(`Generation ${this.releaseId} is not retained for reactivation`)
+    const processConfig = this.config.processes.find((candidate) => candidate.lifecycle.activateCommand !== undefined)
+
+    if (!processConfig) throw new Error(`Generation ${this.releaseId} has no activation lifecycle`)
+    const [coordinator] = this.getProcesses(processConfig.id)
+
+    if (!coordinator) throw new Error(`Generation activation process ${processConfig.id} is not retained for release ${this.releaseId}`)
+    const generationIds = new Set([...this.handoffServiceIds, ...this.nonBlockingDrainIds])
+
+    for (const [id, processInstance] of this.processes) {
+      if (generationIds.has(id) && processInstance !== coordinator.process) await processInstance.reactivateStrict()
+    }
+    await coordinator.process.reactivateStrict()
+    this.state = "active"
+    this.activatedAt = new Date().toISOString()
+    this.drainStartedAt = undefined
+    this.retirementError = undefined
+    this.stoppedAt = undefined
+  }
+
   /**
    * Restarts only the exact processes reconstructed for a committed generation.
    * The caller must prove the durable transition identity before using this path.
    */
   async restartCommittedGeneration() {
+    if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
     this.assertCommittedGenerationRecoverable()
     this.state = "starting"
     try {
@@ -288,7 +314,10 @@ export default class ReleaseGroup extends EventEmitter {
         const instances = this.getProcesses(processConfig.id)
 
         if (instances.length !== processConfig.replicas) throw new Error(`Committed generation ${this.releaseId} is missing process ${processConfig.id}`)
-        for (const {process} of instances) await process.start("deploy", processConfig.lifecycle.activateCommand ? "candidate" : undefined)
+        for (const {process} of instances) {
+          if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
+          await process.start("deploy", processConfig.lifecycle.activateCommand ? "candidate" : undefined)
+        }
 
         if (processConfig.policy === "proxied" && processConfig.port && processConfig.health) {
           await waitForHealth({
@@ -296,6 +325,7 @@ export default class ReleaseGroup extends EventEmitter {
             host: this.config.proxy.upstreamHost,
             port: this.ports[processConfig.id]
           })
+          if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
         }
       }
     } catch (error) {
@@ -564,6 +594,14 @@ export default class ReleaseGroup extends EventEmitter {
     return this.transferredConnections.http + this.transferredConnections.websocket > 0
   }
 
+  /** @returns {ReleaseConnections} Connections physically owned by this daemon listener. */
+  localConnections() {
+    return {
+      http: this.connections.http - this.transferredConnections.http,
+      websocket: this.connections.websocket - this.transferredConnections.websocket
+    }
+  }
+
   /** Pauses only daemon-local connection-dependent retirement at owner handoff. */
   pauseDrainForOwnerHandoff() {
     if (this.state !== "draining") return

package/test/completion.test.js

@@ -41,9 +41,10 @@ test("completion bash prints a sourceable script with commands and option flags"
 
   assert.notEqual(code, 1)
   assert.match(output, /complete -F _rollbridge rollbridge/)
-  assert.match(output, /compgen -W "daemon deploy rollback ensure-daemon status stop restart shutdown validate doctor logs events predeploy-cleanup recover completion"/)
+  assert.match(output, /compgen -W "daemon deploy rollback recover-generation-transition ensure-daemon status stop restart shutdown validate doctor logs events predeploy-cleanup recover completion"/)
   // A command's own options are completed after the command.
   assert.match(output, /deploy\)\n\s+opts="[^"]*--release-path[^"]*"/)
+  assert.match(output, /recover-generation-transition\)\n\s+opts="--config --release-path --release-id --revision --previous-release-id"/)
   assert.match(output, /ensure-daemon\)\n\s+opts="[^"]*--daemon-runtime-path[^"]*"/)
   assert.match(output, /restart\)\n\s+opts="[^"]*--policy[^"]*"/)
 })
@@ -53,7 +54,8 @@ test("completion zsh prints a #compdef script with per-command options", async (
 
   assert.match(output, /^#compdef rollbridge/)
   assert.match(output, /compdef _rollbridge rollbridge/)
-  assert.match(output, /commands=\(daemon deploy rollback ensure-daemon status stop restart shutdown validate doctor logs events predeploy-cleanup recover completion\)/)
+  assert.match(output, /commands=\(daemon deploy rollback recover-generation-transition ensure-daemon status stop restart shutdown validate doctor logs events predeploy-cleanup recover completion\)/)
+  assert.match(output, /recover-generation-transition\) compadd -- --config --release-path --release-id --revision --previous-release-id/)
   assert.match(output, /events\) compadd -- [^\n]*--limit/)
 })
 

package/test/config-examples.test.js

@@ -27,6 +27,7 @@ test("TensorBuzz example config loads", async () => {
       ["web", "proxied"]
     ]
   )
+  assert.equal(config.processes[2].lifecycle.reactivateCommand, "appctl jobs-worker-reactivate --pid $ROLLBRIDGE_PID")
   assert.equal(config.processes[3].env.VELOCIOUS_BACKGROUND_JOBS_PORT, "{{ports.background-jobs-main}}")
 })
 

package/test/config-validation.test.js

@@ -231,6 +231,10 @@ test("validateConfig accepts one durable handoff activation lifecycle and reject
   const invalidType = validateConfig({...base, processes: [base.processes[0], {...base.processes[1], lifecycle: {activateCommand: 5, quietCommand: "jobs retire"}}]})
   assert.ok(invalidType.issues.some((issue) => issue.message === "processes[1].lifecycle.activateCommand must be a string"))
 
+  const emptyCommands = validateConfig({...base, processes: [base.processes[0], {...base.processes[1], lifecycle: {activateCommand: " ", quietCommand: ""}}]})
+  assert.ok(emptyCommands.issues.some((issue) => issue.message === "processes[1].lifecycle.activateCommand must not be empty"))
+  assert.ok(emptyCommands.issues.some((issue) => issue.message === "processes[1].lifecycle.quietCommand must not be empty"))
+
   const missingRetirement = validateConfig({...base, processes: [base.processes[0], {...base.processes[1], lifecycle: {activateCommand: "jobs activate"}}]})
   assert.ok(missingRetirement.issues.some((issue) => /requires lifecycle\.quietCommand/.test(issue.message)))
 
@@ -246,6 +250,24 @@ test("validateConfig accepts one durable handoff activation lifecycle and reject
     {...base.processes[1], id: "jobs-secondary", port: {from: 18200, to: 18299}}
   ]})
   assert.ok(duplicate.issues.some((issue) => /at most one lifecycle\.activateCommand/.test(issue.message)))
+
+  const worker = {
+    command: "run worker",
+    id: "worker",
+    lifecycle: {quietCommand: "worker quiet", reactivateCommand: "worker resume"},
+    nonBlockingDrain: true,
+    policy: "companion"
+  }
+  const pairedWorker = validateConfig({...base, processes: [...base.processes, worker]})
+
+  assert.deepEqual(pairedWorker.issues, [])
+  assert.equal(pairedWorker.config.processes[2].lifecycle.reactivateCommand, "worker resume")
+
+  const unpairedWorker = validateConfig({...base, processes: [...base.processes, {...worker, lifecycle: {quietCommand: "worker quiet"}}]})
+  assert.ok(unpairedWorker.issues.some((issue) => /quietCommand requires lifecycle\.reactivateCommand/.test(issue.message)))
+
+  const unsupportedPlacement = validateConfig({...base, processes: [...base.processes, {...worker, nonBlockingDrain: false}]})
+  assert.ok(unsupportedPlacement.issues.some((issue) => /reactivateCommand.*nonBlockingDrain companion/.test(issue.message)))
 })
 
 test("validateConfig accepts indefinite graceful stop windows", () => {

package/test/fixtures/guardian-recovery-owner.js

@@ -0,0 +1,86 @@
+// @ts-check
+
+import fs from "node:fs"
+import net from "node:net"
+import {spawn} from "node:child_process"
+
+const authorityText = process.env.GUARDIAN_AUTHORITY
+const claimDelayMs = Number(process.env.GUARDIAN_CLAIM_DELAY_MS || 0)
+const descendantPath = process.env.GUARDIAN_DESCENDANT_PATH
+const exitAfterClaim = process.env.GUARDIAN_EXIT_AFTER_CLAIM === "1"
+const markerPath = process.env.GUARDIAN_MARKER_PATH
+const replacementCommittedPath = process.env.GUARDIAN_REPLACEMENT_COMMITTED_PATH
+const replacementPreparedPath = process.env.GUARDIAN_REPLACEMENT_PREPARED_PATH
+const skipReady = process.env.GUARDIAN_SKIP_READY === "1"
+const socketPath = process.env.GUARDIAN_SOCKET_PATH
+const startedLogPath = process.env.GUARDIAN_STARTED_LOG_PATH
+const startedPath = process.env.GUARDIAN_STARTED_PATH
+const token = process.env.GUARDIAN_TOKEN
+
+if (!authorityText || !markerPath || !socketPath || !token || !Number.isInteger(claimDelayMs) || claimDelayMs < 0) {
+  throw new Error("Guardian recovery owner fixture requires authority, marker, socket, token, and a valid claim delay")
+}
+
+let buffer = ""
+/** @type {string | undefined} */
+let replacementId
+let replacementCommitted = false
+
+if (startedPath) fs.writeFileSync(startedPath, `${process.pid}\n`)
+if (startedLogPath) fs.appendFileSync(startedLogPath, `${JSON.stringify({at: Date.now(), pid: process.pid})}\n`)
+if (descendantPath) {
+  const descendant = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], {stdio: "ignore"})
+
+  if (!descendant.pid) throw new Error("Guardian recovery owner fixture descendant did not start")
+  fs.writeFileSync(descendantPath, `${descendant.pid}\n`)
+  descendant.unref()
+}
+const timer = setTimeout(() => {
+  const socket = net.createConnection(socketPath)
+
+  socket.setEncoding("utf8")
+  socket.once("connect", () => {
+    socket.write(`${JSON.stringify({authority: JSON.parse(authorityText), command: "claim-owner", graceMs: 0, id: 1, ownerPid: process.pid, token})}\n`)
+  })
+  socket.on("data", (chunk) => {
+    buffer += chunk
+    let newline = buffer.indexOf("\n")
+
+    while (newline >= 0) {
+      const response = JSON.parse(buffer.slice(0, newline))
+
+      buffer = buffer.slice(newline + 1)
+      if (response.error) throw new Error(String(response.error))
+      if (response.id === 1) {
+        fs.writeFileSync(markerPath, `${process.pid}\n`)
+        if (exitAfterClaim) process.exit(47)
+        if (!skipReady) socket.write(`${JSON.stringify({command: "owner-ready", id: 2, ownerPid: process.pid, token})}\n`)
+      }
+      if (response.event === "replacement-prepared") {
+        replacementId = response.replacementId
+        if (replacementPreparedPath) fs.writeFileSync(replacementPreparedPath, `${replacementId}\n`)
+      }
+      if (response.id === 3) {
+        replacementCommitted = true
+        if (replacementCommittedPath) fs.writeFileSync(replacementCommittedPath, `${replacementId}\n`)
+      }
+      if (response.id === 4) socket.write(`${JSON.stringify({command: "finalize-owner-replacement", id: 5, replacementId, token})}\n`)
+      if (response.id === 5) socket.destroy()
+      newline = buffer.indexOf("\n")
+    }
+  })
+  process.once("SIGUSR1", () => {
+    if (!replacementCommitted) {
+      socket.destroy()
+      return
+    }
+    socket.write(`${JSON.stringify({command: "complete-owner-listener-retirement", id: 4, replacementId, token})}\n`)
+  })
+  process.once("SIGUSR2", () => {
+    if (!replacementId) throw new Error("Guardian recovery owner fixture has no prepared replacement")
+    socket.write(`${JSON.stringify({command: "commit-owner-replacement", id: 3, replacementId, token})}\n`)
+  })
+}, claimDelayMs)
+
+timer.unref()
+setInterval(() => {}, 1000)

package/test/fixtures/pre-split3-process-guardian.js

@@ -212,6 +212,20 @@ async function execute(request, socket) {
   } else if (request.command === "update") {
     if (!request.definition || !request.provenance) throw new Error("Guardian update requires definition and provenance")
     if (record.provenance !== request.previousProvenance) throw new Error(`Guardian provenance mismatch for ${request.key}`)
+    const updateGatePath = request.definition.env?.ROLLBRIDGE_TEST_UPDATE_GATE
+
+    if (updateGatePath) {
+      await fs.writeFile(`${updateGatePath}.waiting`, "waiting\n")
+      while (true) {
+        try {
+          await fs.access(updateGatePath)
+          break
+        } catch (error) {
+          if (!error || typeof error !== "object" || !("code" in error) || error.code !== "ENOENT") throw error
+        }
+        await new Promise((resolve) => setTimeout(resolve, 10))
+      }
+    }
     record.process.updateDefinition({
       ...request.definition,
       lifecycle: request.definition.lifecycle || {drainTimeoutMs: 0},