rollbridge 0.1.55 → 0.1.56

package/changelog.d/20261008-stale-stopped-connection-count.md

@@ -0,0 +1,14 @@
+### Fixed
+
+- Stopped releases no longer carry a stale connection count into owner
+  replacement. When a drain times out the release's local connection count was
+  frozen non-zero, persisted into the daemon state, and replayed into every
+  successor daemon's owner-replace handoff. The successor daemon does not
+  retain the stopped release, so the replay failed the candidate with
+  `Incumbent listener reported unknown release <id>` and deadlocked every
+  subsequent deployment. A release that reaches terminal `stopped` now
+  abandons its local connection count (it owns no live proxy listener),
+  `restore()` no longer re-adopts a persisted count for a stopped release,
+  the serialized owner handoff skips stopped releases, and a non-zero report
+  for a release the daemon does not retain is reconciled to zero with a log
+  line instead of throwing, so a stale count can never wedge the node again.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rollbridge",
-  "version": "0.1.55",
+  "version": "0.1.56",
   "description": "Zero-downtime process supervisor and local traffic switcher for deploy-managed apps.",
   "keywords": [
     "deploy",

package/src/daemon.js

@@ -832,7 +832,22 @@ export default class RollbridgeDaemon {
   setListenerConnectionSource(sourceId, releaseId, connections) {
     const release = this.releases.get(releaseId)
 
-    if (!release && (connections.http > 0 || connections.websocket > 0)) throw new Error(`Incumbent listener reported unknown release ${releaseId}`)
+    if (!release) {
+      if (connections.http + connections.websocket > 0) {
+        // A live proxy connection is always attributed to a release that still listens. A
+        // non-zero report for a release this daemon does not retain is a stale count frozen by
+        // a drain timeout in an earlier owner; accounting for it here would deadlock every
+        // future handoff, so reconcile it to zero instead of failing the owner replacement.
+        this.logger("dropping stale connection state for unretained release", {connections, releaseId, sourceId})
+      }
+      const sourceReleases = this.listenerConnectionSources.get(sourceId)
+
+      if (sourceReleases) {
+        sourceReleases.delete(releaseId)
+        if (sourceReleases.size === 0) this.listenerConnectionSources.delete(sourceId)
+      }
+      return
+    }
     const sourceReleases = this.listenerConnectionSources.get(sourceId) || new Map()
 
     if (connections.http + connections.websocket === 0) sourceReleases.delete(releaseId)
@@ -855,10 +870,23 @@ export default class RollbridgeDaemon {
 
   /** @returns {Record<string, Record<string, {http: number, websocket: number}>>} Exact live listener sources. */
   serializedListenerConnectionSources() {
-    const sources = new Map([...this.listenerConnectionSources].map(([sourceId, releases]) => [sourceId, new Map(releases)]))
+    const sources = new Map()
+
+    for (const [sourceId, releases] of this.listenerConnectionSources) {
+      const retained = new Map()
+
+      for (const [releaseId, connections] of releases) {
+        // A stopped release owns no live listener; its count is a stale drain timeout. Carrying
+        // it into a successor daemon would report a connection the successor cannot attribute.
+        if (this.releases.get(releaseId)?.state === "stopped") continue
+        retained.set(releaseId, connections)
+      }
+      if (retained.size > 0) sources.set(sourceId, retained)
+    }
     const local = new Map()
 
     for (const release of this.releases.values()) {
+      if (release.state === "stopped") continue
       const connections = release.localConnections()
 
       if (connections.http + connections.websocket > 0) local.set(release.releaseId, connections)

package/src/release-group.js

@@ -196,7 +196,9 @@ export default class ReleaseGroup extends EventEmitter {
     this.drainStartedAt = snapshot.drainStartedAt
     this.retirementError = snapshot.retirementError
     this.stoppedAt = snapshot.stoppedAt
-    this.setTransferredConnections(snapshot.connections)
+    // A stopped release no longer owns a proxy listener, so a persisted nonzero count is a
+    // stale drain timeout; re-adopting it would poison this daemon's handoff to the next one.
+    if (snapshot.state !== "stopped") this.setTransferredConnections(snapshot.connections)
 
     for (const processStatus of snapshot.processes) {
       const baseId = processStatus.id.replace(/#\d+$/, "")
@@ -625,6 +627,22 @@ export default class ReleaseGroup extends EventEmitter {
     }
   }
 
+  /**
+   * Abandons every connection this daemon still counts for the release. A release that
+   * reaches terminal `stopped` owns no live proxy listener, so any remaining local count
+   * is a stale drain timeout. Clearing it here (rather than relying on each in-flight
+   * connection to release) keeps the invariant "a stopped release holds no connections"
+   * intact for state persistence, owner-replacement handoff, and future daemons.
+   */
+  abandonLocalConnections() {
+    if (this.state !== "stopped") return
+    if (this.connectionCount === 0 && this.transferredConnections.http + this.transferredConnections.websocket === 0) return
+    this.connections = {http: 0, websocket: 0}
+    this.transferredConnections = {http: 0, websocket: 0}
+    this.connectionCount = 0
+    this.emit("drained")
+  }
+
   /**
    * Reconciles connections still owned by a prior daemon listener.
    * @param {ReleaseConnections} connections - Exact incumbent listener counts.
@@ -721,6 +739,7 @@ export default class ReleaseGroup extends EventEmitter {
     this.state = "stopped"
     this.stoppedAt = new Date().toISOString()
     this.releasePortReservations()
+    this.abandonLocalConnections()
   }
 
   /**
@@ -763,6 +782,7 @@ export default class ReleaseGroup extends EventEmitter {
     this.state = "stopped"
     this.stoppedAt = new Date().toISOString()
     this.releasePortReservations()
+    this.abandonLocalConnections()
   }
 
   /** @returns {Promise<void>} Quiesces every release process without waiting for its drain. */

package/test/owner-replacement.test.js

@@ -1649,10 +1649,13 @@ test("pruned release connection completion closes the incumbent listener session
   expect(closeCount).toBe(1)
   expect(daemon.incumbentListenerControl).toBe(undefined)
 
+  // A nonzero count for a release the daemon no longer retains is a stale drain timeout, not a
+  // live connection: reconcile it to zero and close the session instead of throwing, so the
+  // owner handoff cannot be deadlocked by a pruned generation's frozen count.
   daemon.incumbentListenerControl = controlSession
-  await expect(() => daemon.handleIncumbentListenerEvent({connections: {http: 1, websocket: 0}, event: "owner-connection-state", releaseId: "pruned"}, session)).toThrow(/unknown release pruned/)
-  expect(closeCount).toBe(1)
-  expect(daemon.incumbentListenerControl).toBe(session)
+  expect(() => daemon.handleIncumbentListenerEvent({connections: {http: 1, websocket: 0}, event: "owner-connection-state", releaseId: "pruned"}, session)).not.toThrow()
+  expect(closeCount).toBe(2)
+  expect(daemon.incumbentListenerControl).toBe(undefined)
 })
 
 test("same-authority owner replacement preserves completed activation compensation without replaying hooks", async () => {

package/test/release-group.test.js

@@ -93,4 +93,61 @@ test("committed generation restoration does not start after shutdown begins", as
   await expect(release.restartCommittedGeneration()).rejects.toThrow(/shutting down/)
   expect(starts).toBe(0)
 })
+
+test("stopping a release abandons its connection count instead of persisting a stale drain timeout", async () => {
+  const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}})
+  // In-flight connections whose response never completes (a drain timeout) hold the count.
+  const stuck = [0, 1, 2, 3].map(() => release.retainConnection("http"))
+
+  expect(release.connectionCount).toBe(4)
+
+  await release.stop()
+
+  expect(stuck).toHaveLength(4)
+  expect(release.state).toBe("stopped")
+  expect(release.status().connectionCount).toBe(0)
+  expect(release.localConnections()).toEqual({http: 0, websocket: 0})
+})
+
+test("restoring a stopped release does not re-adopt a stale connection count", async () => {
+  const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}})
+
+  const processStatus = /** @type {import("../src/managed-process.js").ManagedProcessStatus} */ ({
+    children: [],
+    command: "run web",
+    cwd: "/tmp/rel",
+    exitCode: 143,
+    exitSignal: "SIGTERM",
+    id: "web",
+    lastMemoryRestartAt: undefined,
+    lastStartReason: "deploy",
+    logs: [],
+    memoryRestarts: 0,
+    pid: undefined,
+    restarts: 0,
+    rssBytes: undefined,
+    startedAt: "2026-10-08T06:22:23.500Z",
+    state: "stopped",
+    uptimeMs: 900000
+  })
+
+  await release.restore({
+    activatedAt: "2026-10-08T06:22:23.583Z",
+    connectionCount: 14,
+    connections: {http: 14, websocket: 0},
+    drainStartedAt: "2026-10-08T07:58:00.464Z",
+    ports: {web: 4200},
+    processes: [processStatus],
+    releaseId: "v1",
+    releasePath: "/tmp/rel",
+    retirementError: undefined,
+    revision: "v1",
+    state: "stopped",
+    stoppedAt: "2026-10-08T08:28:02.005Z"
+  }, {synchronizeLifecycleRole: false})
+
+  expect(release.state).toBe("stopped")
+  expect(release.status().connectionCount).toBe(0)
+  expect(release.localConnections()).toEqual({http: 0, websocket: 0})
+})
 })

package/test/stale-connection-handoff.test.js

@@ -0,0 +1,84 @@
+// @ts-check
+
+import {describe, expect, test} from "@velocious/testing"
+import RollbridgeDaemon from "../src/daemon.js"
+import ReleaseGroup from "../src/release-group.js"
+import {normalizeConfig} from "../src/config.js"
+
+describe("stale connection handoff", () => {
+
+/**
+ * @param {import("../src/json.js").JsonValue} webProcess - The single proxied process definition.
+ * @returns {import("../src/config.js").RollbridgeConfig} A normalized daemon config.
+ */
+function daemonConfig(webProcess) {
+  return normalizeConfig({
+    application: "stale-connection-handoff-test",
+    control: {path: "/tmp/rollbridge-stale-connection-handoff.sock"},
+    processes: [webProcess],
+    proxy: {host: "127.0.0.1", port: 0}
+  })
+}
+
+const webProcess = {
+  command: "run web",
+  id: "web",
+  policy: "proxied",
+  port: {from: 0, to: 0}
+}
+
+test("the serialized owner handoff never carries a stopped release's stale connection count", () => {
+  const daemon = new RollbridgeDaemon({config: daemonConfig(webProcess), logger: () => {}})
+  const release = new ReleaseGroup({
+    config: daemon.config,
+    logger: () => {},
+    releaseId: "stopped-v1",
+    releasePath: "/tmp/stopped-v1",
+    revision: "stopped-v1"
+  })
+
+  release.state = "stopped"
+  // Frozen drain-timeout count that a prior owner persisted (the 062147 incident shape).
+  release.setTransferredConnections({http: 14, websocket: 0})
+  daemon.releases.set(release.releaseId, release)
+  daemon.setListenerConnectionSource("incumbent-listener", release.releaseId, {http: 14, websocket: 0})
+
+  expect(daemon.serializedListenerConnectionSources()).toEqual({})
+})
+
+test("a successor daemon reconciles a stale report for an unretained release instead of failing the handoff", () => {
+  const logs = /** @type {{message: string, data?: Record<string, import("../src/json.js").JsonValue>}[]} */ ([])
+  const daemon = new RollbridgeDaemon({
+    config: daemonConfig(webProcess),
+    logger: (message, data) => {
+      logs.push({message, data})
+    }
+  })
+
+  // The candidate daemon does not rehydrate the stopped release; the incumbent's handoff
+  // still reports a nonzero count for it.
+  expect(() => daemon.setListenerConnectionSource("incumbent-listener", "062147", {http: 14, websocket: 0})).not.toThrow()
+
+  expect(daemon.serializedListenerConnectionSources()).toEqual({})
+  expect(logs.some((entry) => entry.message === "dropping stale connection state for unretained release")).toBe(true)
+})
+
+test("live release connection counts still cross the owner handoff", () => {
+  const daemon = new RollbridgeDaemon({config: daemonConfig(webProcess), logger: () => {}})
+  const release = new ReleaseGroup({
+    config: daemon.config,
+    logger: () => {},
+    releaseId: "active-v2",
+    releasePath: "/tmp/active-v2",
+    revision: "active-v2"
+  })
+
+  release.state = "active"
+  daemon.releases.set(release.releaseId, release)
+  daemon.setListenerConnectionSource("incumbent-listener", release.releaseId, {http: 3, websocket: 1})
+
+  // The incumbent's live counts must still cross the handoff so the successor can adopt them.
+  expect(daemon.serializedListenerConnectionSources()).toEqual({"incumbent-listener": {"active-v2": {http: 3, websocket: 1}}})
+  expect(release.status().connections).toEqual({http: 3, websocket: 1})
+})
+})