socketduct 0.1.17 → 0.1.19

package/src/reverse/hub.js

@@ -24,6 +24,8 @@ import {
 } from "./common.js"
 
 const DEFAULT_OUTPUT_BUFFER_BYTES = 32 * 1024 * 1024
+const DEFAULT_RESUME_MS = 60_000
+const MANIFEST_PAGE_STREAMS = 64
 
 /**
  * @typedef {{userId: string, username: string}} Principal
@@ -48,6 +50,7 @@ const DEFAULT_OUTPUT_BUFFER_BYTES = 32 * 1024 * 1024
  * @property {number} [handshakeTimeoutMs] - Authentication deadline.
  * @property {number} [maxInputBytesPerConnection] - Pre-authentication byte bound.
  * @property {number} [maxOutputBytesPerConnection] - Per-physical queued hub output bound.
+ * @property {number} [resumeWindowMs] - Detached reverse-stream retention window.
  * @property {import('../protocol/compression.js').CompressionLimits} [compressionLimits] - zstd safety bounds.
  * @typedef {object} HubPhysical
  * @property {import('node:tls').TLSSocket} socket
@@ -63,6 +66,14 @@ const DEFAULT_OUTPUT_BUFFER_BYTES = 32 * 1024 * 1024
  * @property {number} nextGatewayStreamId
  * @property {boolean} authenticated
  * @property {boolean} closing
+ * @property {boolean} retainOnDetach
+ * @property {string | undefined} detachCode
+ * @property {boolean} resumeRequired
+ * @property {number} resumeNextPage
+ * @property {Array<Record<string, unknown>>} resumeItems
+ * @property {boolean} resumeFinal
+ * @property {Array<{streamId: number, bridge: Bridge | undefined, code?: string}>} resumeEntries
+ * @property {ReturnType<typeof setTimeout> | undefined} resumeTimer
  * @property {(() => void) | undefined} clearHandshake
  * @property {HubOutputItem[]} outputQueue
  * @property {number} outputBytes
@@ -76,15 +87,23 @@ const DEFAULT_OUTPUT_BUFFER_BYTES = 32 * 1024 * 1024
  * @property {string} userId
  * @property {number} generation
  * @property {HubPhysical | undefined} physical
+ * @property {Map<number, Bridge>} streams
+ * @property {ReturnType<typeof setTimeout> | undefined} expiryTimer
  * @typedef {object} ClientOwner
  * @property {string} connectionId
  * @property {string} userId
  * @property {number} generation
  * @property {HubPhysical | undefined} physical
+ * @property {Map<number, Bridge>} streams
+ * @property {string | undefined} gatewayId
+ * @property {string | undefined} routeClass
+ * @property {ReturnType<typeof setTimeout> | undefined} expiryTimer
  * @typedef {object} Bridge
  * @property {HubPhysical} client
+ * @property {ClientOwner} clientOwner
  * @property {number} clientStreamId
  * @property {HubPhysical} gateway
+ * @property {GatewayOwner} gatewayOwner
  * @property {number} gatewayStreamId
  * @property {string} identifier
  * @property {Target} target
@@ -97,6 +116,9 @@ const DEFAULT_OUTPUT_BUFFER_BYTES = 32 * 1024 * 1024
  * @property {boolean} gatewayEnded
  * @property {AdaptiveCompressor} clientToGatewayCompressor
  * @property {AdaptiveCompressor} gatewayToClientCompressor
+ * @property {{generation: number, clientToGatewaySequence: number, clientToGatewayAck: number, gatewayToClientSequence: number, gatewayToClientAck: number, localEnded: boolean, remoteEnded: boolean} | undefined} clientResume
+ * @property {{generation: number, clientToGatewaySequence: number, clientToGatewayAck: number, gatewayToClientSequence: number, gatewayToClientAck: number, localEnded: boolean, remoteEnded: boolean} | undefined} gatewayResume
+ * @property {string | undefined} reconciledResume
  */
 
 /** Starts the Server3 reverse hub. */
@@ -108,6 +130,7 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
   const maxIdentities = positiveInteger(options.maxConnectionIdentities, "maxConnectionIdentities", 4096)
   const maxPhysicalConnections = positiveInteger(options.maxPhysicalConnections, "maxPhysicalConnections", 8192)
   const maxOutputBytes = positiveInteger(options.maxOutputBytesPerConnection, "maxOutputBytesPerConnection", DEFAULT_OUTPUT_BUFFER_BYTES)
+  const resumeWindowMs = positiveInteger(options.resumeWindowMs, "resumeWindowMs", DEFAULT_RESUME_MS)
   /** @type {Map<string, GatewayOwner>} */
   const gateways = new Map()
   /** @type {Map<string, ClientOwner>} */
@@ -175,6 +198,14 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
       nextGatewayStreamId: 2,
       authenticated: false,
       closing: false,
+      retainOnDetach: true,
+      detachCode: undefined,
+      resumeRequired: false,
+      resumeNextPage: 0,
+      resumeItems: [],
+      resumeFinal: false,
+      resumeEntries: [],
+      resumeTimer: undefined,
       clearHandshake: undefined,
       outputQueue: [],
       outputBytes: 0,
@@ -204,7 +235,7 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
     socket.on("error", () => {})
     socket.once("close", () => {
       clearTimeout(handshakeTimer)
-      void queue.finally(() => detachPhysical(physical, "TRANSPORT_CLOSED"))
+      void queue.finally(() => detachPhysical(physical, physical.detachCode ?? "TRANSPORT_CLOSED"))
     })
     physical.clearHandshake = () => clearTimeout(handshakeTimer)
   }
@@ -246,6 +277,7 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
     }
     if (frame.type === V3FrameType.PONG) return
     if (frame.type === V3FrameType.GOAWAY) {
+      physical.retainOnDetach = false
       physical.socket.end()
       return
     }
@@ -261,7 +293,12 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
       await selectClientRoute(physical, frame)
       return
     }
-    if (frame.type === V3FrameType.GATEWAY_REGISTER || frame.type === V3FrameType.GATEWAY_REGISTERED || frame.type === V3FrameType.GATEWAY_SELECT || frame.type === V3FrameType.GATEWAY_SELECTED || frame.type === V3FrameType.AUTH || frame.type === V3FrameType.AUTH_OK || frame.type === V3FrameType.AUTH_ERROR || frame.type === V3FrameType.RESUME || frame.type === V3FrameType.RESUMED) throw protocolError("UNSUPPORTED_FRAME")
+    if (physical.resumeRequired && frame.type !== V3FrameType.RESUME) throw protocolError("RESUME_REQUIRED")
+    if (frame.type === V3FrameType.RESUME) {
+      await receiveResumeManifest(physical, frame)
+      return
+    }
+    if (frame.type === V3FrameType.GATEWAY_REGISTER || frame.type === V3FrameType.GATEWAY_REGISTERED || frame.type === V3FrameType.GATEWAY_SELECT || frame.type === V3FrameType.GATEWAY_SELECTED || frame.type === V3FrameType.AUTH || frame.type === V3FrameType.AUTH_OK || frame.type === V3FrameType.AUTH_ERROR || frame.type === V3FrameType.RESUMED) throw protocolError("UNSUPPORTED_FRAME")
     if (physical.role === "client" && frame.type === V3FrameType.OPEN) {
       await openBridge(physical, frame)
       return
@@ -279,13 +316,30 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
     if (existing === undefined && clients.size >= limit) evictDisconnectedClientIdentity()
     if (existing === undefined && clients.size >= limit) throw protocolError("IDENTITY_LIMIT")
     if (existing?.physical !== undefined) terminate(existing.physical, "CONNECTION_REPLACED")
-    clients.set(id, {connectionId: id, userId: principal.userId, generation: physical.generation, physical})
+    if (existing?.expiryTimer !== undefined) clearTimeout(existing.expiryTimer)
+    const owner = existing ?? {
+      connectionId: id,
+      userId: principal.userId,
+      generation: physical.generation,
+      physical: undefined,
+      streams: new Map(),
+      gatewayId: undefined,
+      routeClass: undefined,
+      expiryTimer: undefined
+    }
+    owner.generation = physical.generation
+    owner.physical = physical
+    owner.expiryTimer = undefined
+    physical.streams = owner.streams
+    physical.lastClientStreamId = Math.max(0, ...owner.streams.keys())
+    physical.resumeRequired = existing !== undefined && owner.streams.size > 0
+    clients.set(id, owner)
   }
 
   /** Evicts the oldest disconnected client identity while active owners remain fenced. */
   function evictDisconnectedClientIdentity() {
     for (const [id, owner] of clients) {
-      if (owner.physical !== undefined) continue
+      if (owner.physical !== undefined || owner.streams.size > 0) continue
       clients.delete(id)
       return
     }
@@ -315,16 +369,27 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
     }
     if (existing === undefined && gateways.size >= maxIdentities) throw protocolError("IDENTITY_LIMIT")
     if (existing?.physical !== undefined) terminate(existing.physical, "GATEWAY_REPLACED")
+    if (existing?.expiryTimer !== undefined) clearTimeout(existing.expiryTimer)
     physical.gatewayId = request.gatewayId
-    gateways.set(request.gatewayId, {
+    const owner = existing ?? {
       gatewayId: request.gatewayId,
       connectionId,
       userId: physical.principal.userId,
       generation: physical.generation,
-      physical
-    })
+      physical: undefined,
+      streams: new Map(),
+      expiryTimer: undefined
+    }
+    owner.generation = physical.generation
+    owner.physical = physical
+    owner.expiryTimer = undefined
+    physical.streams = owner.streams
+    physical.nextGatewayStreamId = Math.max(0, ...owner.streams.keys()) + 2
+    physical.resumeRequired = existing !== undefined && owner.streams.size > 0
+    gateways.set(request.gatewayId, owner)
     gatewayRegistrations += 1
-    await sendControl(physical, V3FrameType.GATEWAY_REGISTERED, 0, physical.generation, {gatewayId: request.gatewayId, generation: physical.generation})
+    await sendControl(physical, V3FrameType.GATEWAY_REGISTERED, 0, physical.generation, {gatewayId: request.gatewayId, generation: physical.generation, resumed: physical.resumeRequired})
+    armResumeTimeout(physical)
   }
 
   /** @param {HubPhysical} physical - Authenticated client. @param {import('../protocol/v3-frame.js').V3Frame} frame - Selection frame. */
@@ -334,22 +399,190 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
     const hasGatewayId = Object.hasOwn(request, "gatewayId")
     const hasRouteClass = Object.hasOwn(request, "routeClass")
     if (hasGatewayId === hasRouteClass || physical.principal === undefined) throw protocolError("INVALID_GATEWAY_SELECTION")
+    const clientOwner = clients.get(/** @type {string} */ (physical.connectionId))
+    if (clientOwner === undefined || clientOwner.physical !== physical) throw protocolError("STALE_GENERATION")
     if (hasGatewayId) {
       if (typeof request.gatewayId !== "string" || !GATEWAY_ID.test(request.gatewayId)) throw protocolError("INVALID_GATEWAY_SELECTION")
       if (options.authorizeClientGateway({principal: physical.principal, gatewayId: request.gatewayId}) !== true) throw protocolError("GATEWAY_UNAUTHORIZED")
       const owner = gateways.get(request.gatewayId)
       const known = owner !== undefined || options.knownGatewayIds?.includes(request.gatewayId) === true
       if (!known) throw protocolError("UNKNOWN_GATEWAY_ID")
-      if (owner?.physical === undefined) throw protocolError("GATEWAY_UNAVAILABLE")
+      const retainedSelection = clientOwner.gatewayId === request.gatewayId && clientOwner.streams.size > 0
+      const selectable = owner?.physical !== undefined && !owner.physical.socket.destroyed && !owner.physical.closing && !owner.physical.resumeRequired
+      if (!selectable && !retainedSelection) throw protocolError("GATEWAY_UNAVAILABLE")
+      if (clientOwner.routeClass !== undefined || (clientOwner.gatewayId !== undefined && clientOwner.gatewayId !== request.gatewayId)) throw protocolError("GATEWAY_SELECTION_MISMATCH")
       physical.gatewayId = request.gatewayId
-      await sendControl(physical, V3FrameType.GATEWAY_SELECTED, 0, physical.generation, {gatewayId: request.gatewayId, generation: physical.generation})
+      clientOwner.gatewayId = request.gatewayId
+      await sendControl(physical, V3FrameType.GATEWAY_SELECTED, 0, physical.generation, {gatewayId: request.gatewayId, generation: physical.generation, resumed: physical.resumeRequired})
+      armResumeTimeout(physical)
       return
     }
     if (typeof request.routeClass !== "string" || !ROUTE_CLASS.test(request.routeClass)) throw protocolError("INVALID_ROUTE_CLASS_SELECTION")
     if (options.authorizeClientRouteClass?.({principal: physical.principal, routeClass: request.routeClass}) !== true) throw protocolError("ROUTE_CLASS_UNAUTHORIZED")
     if (options.knownRouteClasses?.includes(request.routeClass) !== true) throw protocolError("UNKNOWN_ROUTE_CLASS")
+    if (clientOwner.gatewayId !== undefined || (clientOwner.routeClass !== undefined && clientOwner.routeClass !== request.routeClass)) throw protocolError("GATEWAY_SELECTION_MISMATCH")
     physical.routeClass = request.routeClass
-    await sendControl(physical, V3FrameType.GATEWAY_SELECTED, 0, physical.generation, {routeClass: request.routeClass, generation: physical.generation})
+    clientOwner.routeClass = request.routeClass
+    await sendControl(physical, V3FrameType.GATEWAY_SELECTED, 0, physical.generation, {routeClass: request.routeClass, generation: physical.generation, resumed: physical.resumeRequired})
+    armResumeTimeout(physical)
+  }
+
+  /** @param {HubPhysical} physical - Reconnecting reverse peer. @param {import('../protocol/v3-frame.js').V3Frame} frame - Resume manifest page. */
+  async function receiveResumeManifest(physical, frame) {
+    if (!physical.resumeRequired || physical.resumeFinal || frame.streamId !== 0 || frame.sequence !== physical.generation || frame.flags !== 0) throw protocolError("INVALID_RESUME_MANIFEST")
+    const manifest = decodeV3Control(frame.payload)
+    if (manifest.page !== physical.resumeNextPage || !Array.isArray(manifest.streams) || manifest.streams.length > MANIFEST_PAGE_STREAMS || physical.resumeItems.length + manifest.streams.length > maxStreams) throw protocolError("INVALID_RESUME_MANIFEST")
+    physical.resumeNextPage += 1
+    physical.resumeItems.push(...manifest.streams.filter((item) => typeof item === "object" && item !== null && !Array.isArray(item)))
+    if (manifest.final !== true) return
+    const presented = new Set()
+    physical.resumeEntries = []
+    for (const value of physical.resumeItems) {
+      const item = parseResumeItem(value)
+      if (presented.has(item.streamId)) throw protocolError("DUPLICATE_RESUME_STREAM")
+      presented.add(item.streamId)
+      const bridge = physical.streams.get(item.streamId)
+      if (bridge === undefined) {
+        physical.resumeEntries.push({streamId: item.streamId, bridge: undefined, code: "RESUME_SESSION_LOST"})
+        continue
+      }
+      if (bridge.identifier !== item.identifier) {
+        closeBridge(bridge, "RESUME_IDENTITY_MISMATCH", physical)
+        physical.resumeEntries.push({streamId: item.streamId, bridge: undefined, code: "RESUME_IDENTITY_MISMATCH"})
+        continue
+      }
+      if (physical.role === "client") {
+        if (bridge.clientOwner.physical !== physical) throw protocolError("STALE_GENERATION")
+        bridge.client = physical
+        bridge.clientResume = {generation: physical.generation, ...item}
+      } else {
+        if (bridge.gatewayOwner.physical !== physical) throw protocolError("STALE_GENERATION")
+        bridge.gateway = physical
+        bridge.gatewayResume = {generation: physical.generation, ...item}
+      }
+      physical.resumeEntries.push({streamId: item.streamId, bridge})
+    }
+    for (const bridge of [...physical.streams.values()]) {
+      const streamId = physical.role === "client" ? bridge.clientStreamId : bridge.gatewayStreamId
+      if (!presented.has(streamId)) closeBridge(bridge, "RESUME_STREAM_OMITTED", physical)
+    }
+    physical.resumeItems = []
+    physical.resumeFinal = true
+    await tryCompleteResumes()
+  }
+
+  /** Completes physical resume responses only after both logical endpoints have declared durable positions. */
+  async function tryCompleteResumes() {
+    while (true) {
+      const candidates = [...physicals].filter((physical) => physical.resumeRequired && physical.resumeFinal && !physical.socket.destroyed && !physical.resumeEntries.some(({bridge}) => bridge !== undefined && bridgeStillOwned(bridge) && (bridge.clientResume === undefined || bridge.gatewayResume === undefined)))
+      const readySet = new Set(candidates)
+      let pruned
+      do {
+        pruned = false
+        for (const physical of [...readySet]) {
+          const hasUnreadyPeer = physical.resumeEntries.some(({bridge}) => bridge !== undefined && bridgeStillOwned(bridge) && (!readySet.has(bridge.client) || !readySet.has(bridge.gateway)))
+          if (!hasUnreadyPeer) continue
+          readySet.delete(physical)
+          pruned = true
+        }
+      } while (pruned)
+      const ready = [...readySet]
+      if (ready.length === 0) return
+      for (const physical of ready) {
+        clearResumeTimeout(physical)
+        physical.resumeFinal = false
+        physical.resumeRequired = false
+      }
+      /** @type {Promise<boolean>[]} */
+      const deliveries = []
+      for (const physical of ready) {
+        /** @type {Array<Record<string, unknown>>} */
+        const results = []
+        for (const entry of physical.resumeEntries) {
+          const bridge = entry.bridge
+          if (bridge === undefined || !bridgeStillOwned(bridge)) {
+            results.push({streamId: entry.streamId, status: "reset", code: entry.code ?? "RESUME_SESSION_LOST"})
+            continue
+          }
+          try {
+            reconcileBridge(bridge)
+          } catch {
+            closeBridge(bridge, "RESUME_STATE_MISMATCH")
+            results.push({streamId: entry.streamId, status: "reset", code: "RESUME_STATE_MISMATCH"})
+            continue
+          }
+          results.push({
+            streamId: entry.streamId,
+            status: "resumed",
+            clientToGatewaySequence: bridge.clientToGatewaySequence,
+            clientToGatewayAck: bridge.clientToGatewayAck,
+            gatewayToClientSequence: bridge.gatewayToClientSequence,
+            gatewayToClientAck: bridge.gatewayToClientAck,
+            clientEnded: bridge.clientEnded,
+            gatewayEnded: bridge.gatewayEnded,
+            opened: bridge.state !== "opening"
+          })
+        }
+        const pageCount = Math.max(1, Math.ceil(results.length / MANIFEST_PAGE_STREAMS))
+        for (let page = 0; page < pageCount; page += 1) {
+          deliveries.push(sendControl(physical, V3FrameType.RESUMED, 0, physical.generation, {
+            page,
+            final: page === pageCount - 1,
+            streams: results.slice(page * MANIFEST_PAGE_STREAMS, (page + 1) * MANIFEST_PAGE_STREAMS)
+          }))
+        }
+        physical.resumeEntries = []
+        physical.resumeNextPage = 0
+      }
+      await Promise.all(deliveries)
+    }
+  }
+
+  /** @param {Bridge} bridge - Retained bridge. @returns {boolean} Whether both owners still retain it. */
+  function bridgeStillOwned(bridge) {
+    return bridge.clientOwner.streams.get(bridge.clientStreamId) === bridge && bridge.gatewayOwner.streams.get(bridge.gatewayStreamId) === bridge
+  }
+
+  /** Bounds an authenticated peer that reconnects but never completes its resume manifest. */
+  function armResumeTimeout(/** @type {HubPhysical} */ physical) {
+    clearResumeTimeout(physical)
+    if (!physical.resumeRequired || physical.socket.destroyed) return
+    physical.resumeTimer = setTimeout(() => terminate(physical, "RESUME_WINDOW_EXPIRED"), resumeWindowMs)
+    physical.resumeTimer.unref?.()
+  }
+
+  /** Clears one physical resume deadline. */
+  function clearResumeTimeout(/** @type {HubPhysical} */ physical) {
+    if (physical.resumeTimer !== undefined) clearTimeout(physical.resumeTimer)
+    physical.resumeTimer = undefined
+  }
+
+  /** Reconciles one bridge to receiver-committed byte positions for a fresh physical generation. */
+  function reconcileBridge(/** @type {Bridge} */ bridge) {
+    const client = bridge.clientResume
+    const gateway = bridge.gatewayResume
+    if (client === undefined || gateway === undefined) throw protocolError("RESUME_STATE_MISMATCH")
+    const key = `${client.generation}:${gateway.generation}`
+    if (bridge.reconciledResume === key) return
+    if (gateway.clientToGatewaySequence > client.clientToGatewaySequence || client.clientToGatewayAck > gateway.clientToGatewayAck || client.gatewayToClientSequence > gateway.gatewayToClientSequence || gateway.gatewayToClientAck > client.gatewayToClientAck) throw protocolError("RESUME_STATE_MISMATCH")
+    if ((gateway.remoteEnded && !client.localEnded) || (client.remoteEnded && !gateway.localEnded)) throw protocolError("RESUME_STATE_MISMATCH")
+    bridge.clientToGatewaySequence = gateway.clientToGatewaySequence
+    bridge.clientToGatewayAck = gateway.clientToGatewayAck
+    bridge.gatewayToClientSequence = client.gatewayToClientSequence
+    bridge.gatewayToClientAck = client.gatewayToClientAck
+    bridge.clientEnded = gateway.remoteEnded
+    bridge.gatewayEnded = client.remoteEnded
+    bridge.clientToGatewayCompressor = new AdaptiveCompressor()
+    bridge.gatewayToClientCompressor = new AdaptiveCompressor()
+    bridge.reconciledResume = key
+  }
+
+  /** @param {Record<string, unknown>} value - Resume manifest entry. */
+  function parseResumeItem(value) {
+    const item = /** @type {{streamId?: unknown, identifier?: unknown, clientToGatewaySequence?: unknown, clientToGatewayAck?: unknown, gatewayToClientSequence?: unknown, gatewayToClientAck?: unknown, localEnded?: unknown, remoteEnded?: unknown}} */ (value)
+    const sequences = [item.clientToGatewaySequence, item.clientToGatewayAck, item.gatewayToClientSequence, item.gatewayToClientAck]
+    if (!Number.isInteger(item.streamId) || /** @type {number} */ (item.streamId) <= 0 || typeof item.identifier !== "string" || !STABLE_IDENTIFIER.test(item.identifier) || !sequences.every((sequence) => Number.isSafeInteger(sequence) && /** @type {number} */ (sequence) >= 0) || /** @type {number} */ (item.clientToGatewayAck) > /** @type {number} */ (item.clientToGatewaySequence) || /** @type {number} */ (item.gatewayToClientAck) > /** @type {number} */ (item.gatewayToClientSequence) || typeof item.localEnded !== "boolean" || typeof item.remoteEnded !== "boolean") throw protocolError("INVALID_RESUME_MANIFEST")
+    return /** @type {{streamId: number, identifier: string, clientToGatewaySequence: number, clientToGatewayAck: number, gatewayToClientSequence: number, gatewayToClientAck: number, localEnded: boolean, remoteEnded: boolean}} */ (item)
   }
 
   /** @param {HubPhysical} client - Client physical. @param {import('../protocol/v3-frame.js').V3Frame} frame - OPEN. */
@@ -400,7 +633,7 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
     }
     const gatewayOwner = gateways.get(gatewayId)
     const gateway = gatewayOwner?.physical
-    if (gatewayOwner === undefined || gateway === undefined || gateway.socket.destroyed || gateway.role !== "gateway" || gateway.gatewayId !== gatewayId || gatewayOwner.gatewayId !== gatewayId || gatewayOwner.userId !== gateway.principal?.userId) {
+    if (gatewayOwner === undefined || gateway === undefined || gateway.socket.destroyed || gateway.closing || gateway.resumeRequired || gateway.role !== "gateway" || gateway.gatewayId !== gatewayId || gatewayOwner.gatewayId !== gatewayId || gatewayOwner.userId !== gateway.principal?.userId) {
       rejectedOpens += 1
       const known = gatewayOwner !== undefined || options.knownGatewayIds?.includes(gatewayId) === true
       await sendControl(client, V3FrameType.RESET, frame.streamId, 0, {code: known ? "GATEWAY_UNAVAILABLE" : "UNKNOWN_GATEWAY_ID"})
@@ -412,11 +645,15 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
       return
     }
     gateway.nextGatewayStreamId += 2
+    const clientOwner = clients.get(/** @type {string} */ (client.connectionId))
+    if (clientOwner === undefined || clientOwner.physical !== client) throw protocolError("STALE_GENERATION")
     /** @type {Bridge} */
     const bridge = {
       client,
+      clientOwner,
       clientStreamId: frame.streamId,
       gateway,
+      gatewayOwner,
       gatewayStreamId,
       identifier: request.identifier,
       target,
@@ -428,10 +665,13 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
       clientEnded: false,
       gatewayEnded: false,
       clientToGatewayCompressor: new AdaptiveCompressor(),
-      gatewayToClientCompressor: new AdaptiveCompressor()
+      gatewayToClientCompressor: new AdaptiveCompressor(),
+      clientResume: undefined,
+      gatewayResume: undefined,
+      reconciledResume: undefined
     }
-    client.streams.set(frame.streamId, bridge)
-    gateway.streams.set(gatewayStreamId, bridge)
+    clientOwner.streams.set(frame.streamId, bridge)
+    gatewayOwner.streams.set(gatewayStreamId, bridge)
     bridges += 1
     forwardControl(gateway, V3FrameType.OPEN, gatewayStreamId, 0, {identifier: request.identifier, name: request.name, target, initialWindow})
   }
@@ -513,8 +753,8 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
 
   /** @param {Bridge} bridge - Stream. */
   function removeBridge(bridge) {
-    const existed = bridge.client.streams.delete(bridge.clientStreamId)
-    bridge.gateway.streams.delete(bridge.gatewayStreamId)
+    const existed = bridge.clientOwner.streams.delete(bridge.clientStreamId)
+    bridge.gatewayOwner.streams.delete(bridge.gatewayStreamId)
     if (existed) bridges = Math.max(0, bridges - 1)
   }
 
@@ -535,10 +775,12 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
     removeBridge(bridge)
   }
 
-  /** @param {HubPhysical} physical - Connection. @param {string} code - Reason. */
-  function terminate(physical, code) {
+  /** @param {HubPhysical} physical - Connection. @param {string} code - Reason. @param {boolean} [retainStreams] - Preserve logical streams for paired resume. */
+  function terminate(physical, code, retainStreams = false) {
     if (physical.closing) return
     physical.closing = true
+    physical.retainOnDetach = retainStreams
+    physical.detachCode = code
     const forceTimer = setTimeout(() => physical.socket.destroy(), 1_000)
     forceTimer.unref?.()
     void sendControl(physical, V3FrameType.GOAWAY, 0, 0, {code}, true).finally(() => {
@@ -551,23 +793,57 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
   function detachPhysical(physical, code) {
     if (!physicals.delete(physical)) return
     physical.closing = true
+    clearResumeTimeout(physical)
     discardQueuedOutput(physical)
-    for (const bridge of [...physical.streams.values()]) closeBridge(bridge, physical.role === "gateway" ? "GATEWAY_DISCONNECTED" : code, physical)
     if (physical.gatewayId !== undefined) {
       const owner = gateways.get(physical.gatewayId)
       if (physical.role === "gateway" && owner?.physical === physical) {
         owner.physical = undefined
-        for (const client of [...physicals]) {
-          if (client.role === "client" && client.gatewayId === physical.gatewayId) terminate(client, "GATEWAY_DISCONNECTED")
+        if (!physical.retainOnDetach || closing) {
+          for (const bridge of [...owner.streams.values()]) closeBridge(bridge, code, physical)
+          gateways.delete(owner.gatewayId)
+        } else {
+          scheduleOwnerExpiry(owner, "gateway")
+          const clientsToReconnect = new Set([...owner.streams.values()].map((bridge) => bridge.clientOwner.physical).filter((peer) => peer !== undefined))
+          for (const client of clientsToReconnect) terminate(/** @type {HubPhysical} */ (client), "GATEWAY_DISCONNECTED", true)
         }
       }
     }
     if (physical.role === "client" && physical.connectionId !== undefined) {
       const owner = clients.get(physical.connectionId)
-      if (owner?.physical === physical) owner.physical = undefined
+      if (owner?.physical === physical) {
+        owner.physical = undefined
+        if (!physical.retainOnDetach || closing) {
+          for (const bridge of [...owner.streams.values()]) closeBridge(bridge, code, physical)
+          clients.delete(owner.connectionId)
+        } else {
+          scheduleOwnerExpiry(owner, "client")
+          const gatewaysToReconnect = new Set([...owner.streams.values()].map((bridge) => bridge.gatewayOwner.physical).filter((peer) => peer !== undefined))
+          for (const gateway of gatewaysToReconnect) terminate(/** @type {HubPhysical} */ (gateway), "CLIENT_DISCONNECTED", true)
+        }
+      }
     }
   }
 
+  /** @param {GatewayOwner | ClientOwner} owner - Detached identity. @param {'gateway' | 'client'} role - Owner role. */
+  function scheduleOwnerExpiry(owner, role) {
+    if (owner.expiryTimer !== undefined) clearTimeout(owner.expiryTimer)
+    owner.expiryTimer = setTimeout(() => {
+      owner.expiryTimer = undefined
+      if (owner.physical !== undefined) return
+      for (const bridge of [...owner.streams.values()]) closeBridge(bridge, "RESUME_WINDOW_EXPIRED")
+      if (role === "gateway") {
+        const gatewayOwner = /** @type {GatewayOwner} */ (owner)
+        if (gateways.get(gatewayOwner.gatewayId) === gatewayOwner) gateways.delete(gatewayOwner.gatewayId)
+      } else {
+        const clientOwner = /** @type {ClientOwner} */ (owner)
+        if (clients.get(clientOwner.connectionId) === clientOwner) clients.delete(clientOwner.connectionId)
+      }
+      void tryCompleteResumes()
+    }, resumeWindowMs)
+    owner.expiryTimer.unref?.()
+  }
+
   /** @param {HubPhysical} physical - Destination. @param {Buffer} bytes - Encoded frame. @param {boolean} [allowClosing] - Allow a final GOAWAY after closing starts. @returns {Promise<boolean>} Delivery result. */
   function enqueueOutput(physical, bytes, allowClosing = false) {
     if (physical.socket.destroyed || (physical.closing && !allowClosing)) return Promise.resolve(false)
@@ -619,6 +895,8 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
   /** @param {HubPhysical} physical - Destination whose output transport failed. @param {string} [code] - Terminal code. */
   function abortOutput(physical, code = "OUTPUT_BACKPRESSURE_LIMIT") {
     physical.closing = true
+    physical.retainOnDetach = false
+    physical.detachCode ??= code
     discardQueuedOutput(physical)
     if (!physical.socket.destroyed) physical.socket.destroy(protocolError(code))
   }
@@ -647,7 +925,7 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
       queuedOutputBytes: [...physicals].reduce((total, physical) => total + physical.outputBytes, 0),
       retainedClientIdentities: clients.size,
       registeredGateways: gateways.size,
-      healthyGateways: [...gateways.values()].filter((owner) => owner.physical !== undefined).length,
+      healthyGateways: [...gateways.values()].filter((owner) => owner.physical !== undefined && !owner.physical.socket.destroyed && !owner.physical.closing && !owner.physical.resumeRequired).length,
       gatewayRegistrations,
       rejectedRegistrations,
       rejectedOpens,
@@ -667,6 +945,8 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
       for (const {socket} of pendingTls.values()) socket.destroy()
       for (const physical of [...physicals]) terminate(physical, "HUB_SHUTDOWN")
       await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve(undefined)))
+      for (const owner of gateways.values()) if (owner.expiryTimer !== undefined) clearTimeout(owner.expiryTimer)
+      for (const owner of clients.values()) if (owner.expiryTimer !== undefined) clearTimeout(owner.expiryTimer)
       gateways.clear()
       clients.clear()
     }
@@ -690,5 +970,6 @@ function validateOptions(options) {
   if (options.knownRouteClasses !== undefined && (options.knownRouteClasses.some((routeClass) => typeof routeClass !== "string" || !ROUTE_CLASS.test(routeClass)) || new Set(options.knownRouteClasses).size !== options.knownRouteClasses.length)) throw new TypeError("knownRouteClasses must contain unique valid route classes")
   const frame = positiveInteger(options.maxFrameBytes, "maxFrameBytes", DEFAULT_FRAME_BYTES)
   const window = positiveInteger(options.initialWindowBytes, "initialWindowBytes", DEFAULT_WINDOW)
+  positiveInteger(options.resumeWindowMs, "resumeWindowMs", DEFAULT_RESUME_MS)
   if (frame > window) throw new RangeError("maxFrameBytes must not exceed initialWindowBytes")
 }

package/src/reverse/runtime.js

@@ -51,6 +51,7 @@ export async function startReverseHubRuntime(config) {
       resolveOpenGateways: ({principal, routeClass, target}) => [...(routeClassGateways.get(formatRouteClassRoute(principal.username, routeClass, target)) ?? [])]
     }),
     authorizeOpen: ({principal, gatewayId, target}) => authorizedRoutes.has(formatRoute(principal.username, gatewayId, target)),
+    resumeWindowMs: config.resumeWindowMs,
     handshakeTimeoutMs: config.handshakeTimeoutMs,
     maxInputBytesPerConnection: config.maxInputBytesPerConnection,
     compressionLimits: compressionLimits(config)