socketduct 0.1.18 → 0.1.19

package/src/reverse/client.js

@@ -70,6 +70,8 @@ const MANIFEST_PAGE_STREAMS = 64
  * @property {{host: string, port: number}} target
  * @property {VirtualSocket} socket
  * @property {'opening' | 'open' | 'half-closed' | 'closed' | 'reset'} state
+ * @property {boolean} opened
+ * @property {boolean} openingSent
  * @property {number} sendSequence
  * @property {number} transmittedSequence
  * @property {number} sendAck
@@ -77,8 +79,9 @@ const MANIFEST_PAGE_STREAMS = 64
  * @property {number} sendWindowLimit
  * @property {number} receiveSequence
  * @property {number} receiveAck
- * @property {number | undefined} pendingReceiveAck
+ * @property {Array<{sequence: number, remaining: number}>} receiveFrames
  * @property {boolean} remoteEnded
+ * @property {boolean} remoteEndPending
  * @property {number} bufferedBytes
  * @property {ClientChunk[]} replay
  * @property {AdaptiveCompressor} compressor
@@ -93,6 +96,29 @@ const MANIFEST_PAGE_STREAMS = 64
  * @property {number} spoolReservedBytes
  * @property {Promise<void> | undefined} spoolRemovalPromise
  * @property {boolean} spoolCleanupFailureRecorded
+ * @property {boolean} ownsSpool
+ * @property {boolean} recoveredObject
+ * @typedef {object} ReverseClientRecoveryState
+ * @property {number} version
+ * @property {'reverse-client'} role
+ * @property {string} connectionId
+ * @property {number} generation
+ * @property {string} [gatewayId]
+ * @property {string} [routeClass]
+ * @property {number} streamId
+ * @property {string} identifier
+ * @property {string} name
+ * @property {{host: string, port: number}} target
+ * @property {'opening' | 'open' | 'half-closed'} state
+ * @property {boolean} opened
+ * @property {true} openingSent
+ * @property {number} sendSequence
+ * @property {number} sendAck
+ * @property {number} receiveSequence
+ * @property {number} receiveAck
+ * @property {boolean} localEnded
+ * @property {boolean} remoteEnded
+ * @property {boolean} endSent
  */
 
 /** Server3/TensorBuzz-facing virtual socket connection to an exact gateway or server-resolved route class. */
@@ -106,6 +132,7 @@ export class ReverseGatewayConnection extends EventEmitter {
   /** @type {number} */ #generation
   /** @type {string | undefined} */ #codec
   /** @type {boolean} */ #closing = false
+  /** @type {boolean} */ #preserveSpools = false
   /** @type {boolean} */ #fatal = false
   /** @type {Error | undefined} */ #fatalError
   /** @type {boolean} */ #referenced = true
@@ -127,6 +154,7 @@ export class ReverseGatewayConnection extends EventEmitter {
   /** @type {number} */ #wireDataBytes = 0
   /** @type {number} */ #lastProgressedStreamId = 0
   /** @type {Promise<void>} */ #openQueue = Promise.resolve()
+  /** @type {Promise<void>} */ #physicalFrameQueue = Promise.resolve()
   /** @type {SessionSpool} */ #spool
   /** @type {number} */ #spoolBytes = 0
   /** @type {number} */ #spoolCleanupFailures = 0
@@ -163,15 +191,45 @@ export class ReverseGatewayConnection extends EventEmitter {
 
   /** Allocates one stream on the already-warm gateway transport. */
   createConnection(/** @type {import('../client/multiplexed-connection.js').VirtualConnectionOptions} */ options, /** @type {((error: Error | null, socket: VirtualSocket) => void) | undefined} */ callback = undefined) {
+    if (options?.recover === true) throw new TypeError("Durable recovery must use registerRecoveredConnection()")
+    return this.#createConnection(options, callback, false)
+  }
+
+  /** Registers one durable stream without starting a physical connection. */
+  registerRecoveredConnection(/** @type {import('../client/multiplexed-connection.js').VirtualConnectionOptions} */ options, /** @type {((error: Error | null, socket: VirtualSocket) => void) | undefined} */ callback = undefined) {
+    if (options?.recover !== true) throw new TypeError("registerRecoveredConnection requires recover: true")
+    if (this.#options.connectionId === undefined) throw new TypeError("registerRecoveredConnection requires an explicit stable connectionId")
+    if (this.#options.spoolDirectory === undefined) throw new TypeError("registerRecoveredConnection requires an explicit persistent spoolDirectory")
+    if (this.#physicalAttempts > 0) throw new Error("Recovered streams must be registered before any physical connection attempt")
+    return this.#createConnection(options, callback, true)
+  }
+
+  /** Creates a fresh or synchronously recovered reverse stream. */
+  #createConnection(/** @type {import('../client/multiplexed-connection.js').VirtualConnectionOptions} */ options, /** @type {((error: Error | null, socket: VirtualSocket) => void) | undefined} */ callback, /** @type {boolean} */ recovering) {
     if (this.#closing) throw closedError()
-    if (this.#state !== "ready") throw Object.assign(new Error("Reverse gateway member is not ready"), {code: "SOCKETDUCT_TRANSPORT_SET_NOT_READY"})
+    if (!recovering && this.#state !== "ready") throw Object.assign(new Error("Reverse gateway member is not ready"), {code: "SOCKETDUCT_TRANSPORT_SET_NOT_READY"})
     if (this.#streams.size >= (this.#options.maxStreams ?? DEFAULT_MAX_STREAMS)) throw new Error("Maximum reverse virtual stream count exceeded")
     const target = parseTarget(options?.target ?? {host: options?.host, port: options?.port})
     if (target === undefined) throw new TypeError("Reverse virtual connection target host and port are required")
-    const identifier = options.identifier ?? randomBytes(24).toString("base64url")
+    const identifier = recovering ? options.identifier : (options.identifier ?? randomBytes(24).toString("base64url"))
     if (typeof identifier !== "string" || !STABLE_IDENTIFIER.test(identifier)) throw new TypeError("Reverse virtual stream identifier must be 8-128 URL-safe characters")
-    const id = this.#allocateStreamId()
-    const name = options.sessionName ?? `socketduct-reverse-${id}`
+    const recovery = recovering
+      ? validateRecoveryState(this.#spool.readRecoveryStateSync(identifier, "reverse-client-v3"), {
+        connectionId: this.#connectionId,
+        identifier,
+        target,
+        selection: this.#routeIdentityFields(),
+        initialWindow: this.#initialWindow(),
+        ...(options.sessionName === undefined ? {} : {sessionName: options.sessionName})
+      })
+      : undefined
+    const chunks = recovery === undefined ? [] : this.#spool.loadChunksSync(identifier, "reverse-client-v3", recovery.sendAck, recovery.sendSequence)
+    if (recovery !== undefined) validateRecoveredChunks(recovery, chunks, this.#options.maxStreamBufferBytes ?? DEFAULT_STREAM_BUFFER_BYTES, this.#maxFrameBytes())
+    const bufferedBytes = chunks.reduce((sum, chunk) => sum + chunk.data.length, 0)
+    if (this.#bufferedBytes + bufferedBytes > (this.#options.maxConnectionBufferBytes ?? DEFAULT_CONNECTION_BUFFER_BYTES)) throw new Error("Recovered reverse data exceeds the connection buffer limit")
+    const id = recovery?.streamId ?? this.#allocateStreamId()
+    if (this.#streams.has(id)) throw new Error(`Reverse v3 stream ID ${id} is already active`)
+    const name = options.sessionName ?? recovery?.name ?? `socketduct-reverse-${id}`
     if (typeof name !== "string" || name.length === 0 || name.length > 256) throw new TypeError("Reverse virtual stream name must be 1-256 characters")
     const socket = new VirtualSocket(/** @type {import('../client/multiplexed-connection.js').MultiplexedSocketductConnection} */ (/** @type {unknown} */ (this)), id, {
       ...target,
@@ -182,55 +240,78 @@ export class ReverseGatewayConnection extends EventEmitter {
     /** @type {ReverseClientStream} */
     const stream = {
       id, identifier, name, target, socket,
-      state: "opening",
-      sendSequence: 0,
-      transmittedSequence: 0,
-      sendAck: 0,
+      state: recovery?.state ?? "opening",
+      opened: recovery?.opened ?? false,
+      openingSent: recovery?.openingSent ?? false,
+      sendSequence: recovery?.sendSequence ?? 0,
+      transmittedSequence: recovery?.sendAck ?? 0,
+      sendAck: recovery?.sendAck ?? 0,
       sendWindow: this.#sendWindowLimit,
       sendWindowLimit: this.#sendWindowLimit,
-      receiveSequence: 0,
-      receiveAck: 0,
-      pendingReceiveAck: undefined,
-      remoteEnded: false,
-      bufferedBytes: 0,
-      replay: [],
+      // Bytes buffered only in the dead VirtualSocket were not durably
+      // delivered. Resume inbound delivery from the persisted ACK boundary.
+      receiveSequence: recovery?.receiveAck ?? 0,
+      receiveAck: recovery?.receiveAck ?? 0,
+      receiveFrames: [],
+      remoteEnded: recovery?.remoteEnded ?? false,
+      remoteEndPending: recovery?.remoteEnded ?? false,
+      bufferedBytes,
+      replay: chunks.map((chunk) => ({...chunk, sent: false})),
       compressor: new AdaptiveCompressor(),
       pumping: false,
       pendingWrite: undefined,
       progressingWrite: false,
-      localEnded: false,
-      endSent: false,
+      localEnded: recovery?.localEnded ?? false,
+      endSent: recovery?.endSent ?? false,
       endCallback: undefined,
       spoolReady: Promise.resolve(),
       spoolQueue: Promise.resolve(),
-      spoolReservedBytes: 0,
+      spoolReservedBytes: bufferedBytes,
       spoolRemovalPromise: undefined,
-      spoolCleanupFailureRecorded: false
+      spoolCleanupFailureRecorded: false,
+      ownsSpool: recovery !== undefined,
+      recoveredObject: recovery !== undefined
+    }
+    if (recovery !== undefined) {
+      this.#options.aggregateBudget?.reserveData(bufferedBytes)
+      this.#bufferedBytes += bufferedBytes
+      this.#spoolBytes += bufferedBytes
+      this.#nextStreamId = Math.max(this.#nextStreamId, id + 2)
+      this.#generation = Math.max(this.#generation, recovery.generation)
     }
     this.#streams.set(id, stream)
-    stream.spoolReady = this.#spool.createNew({sessionId: identifier, ownerId: "reverse-client-v3"})
-      .then(() => this.#spool.writeRecoveryState(identifier, this.#recoveryState(stream)))
+    stream.spoolReady = recovery === undefined
+      ? this.#spool.createNew({sessionId: identifier, ownerId: "reverse-client-v3"})
+        .then(() => {
+          stream.ownsSpool = true
+          return this.#spool.writeRecoveryState(identifier, this.#recoveryState(stream))
+        })
+      : this.#spool.truncateAcked(identifier, recovery.sendAck)
     stream.spoolQueue = stream.spoolReady.catch(() => {})
     logDiagnostic(this.#options.logger, "virtual_stream.opening", {
       streamId: id,
       generation: this.#generation,
       ...this.#routeIdentityFields(),
-      recovered: false
+      recovered: recovery !== undefined
     })
     if (callback !== undefined) installCallback(socket, callback)
-    const opening = this.#openQueue
-      .then(() => stream.spoolReady)
-      .then(() => this.#streams.get(id) === stream ? this.#sendControl(V3FrameType.OPEN, id, 0, {...this.#routeIdentityFields(), identifier, name, target, initialWindow: this.#initialWindow()}) : undefined)
-    this.#openQueue = opening.then(() => undefined, () => undefined)
-    void opening.catch((error) => socket._receiveError(asError(error)))
+    if (recovery === undefined) {
+      const opening = this.#openQueue
+        .then(() => stream.spoolReady)
+        .then(async () => {
+          if (this.#streams.get(id) !== stream) return
+          stream.openingSent = true
+          await this.#queueSpoolOperation(stream, () => this.#spool.writeRecoveryState(identifier, this.#recoveryState(stream)))
+          if (this.#streams.get(id) === stream) await this.#sendControl(V3FrameType.OPEN, id, 0, {...this.#routeIdentityFields(), identifier, name, target, initialWindow: this.#initialWindow()})
+        })
+      this.#openQueue = opening.then(() => undefined, () => undefined)
+      void opening.catch((error) => socket._receiveError(asError(error)))
+    } else {
+      void stream.spoolReady.catch((error) => socket._receiveError(asError(error)))
+    }
     return socket
   }
 
-  /** Process replacement cannot inherit the live reverse target/app socket pair. */
-  registerRecoveredConnection() {
-    throw Object.assign(new Error("Reverse gateway process recovery is unsupported; live streams resume only within the original process"), {code: "REVERSE_STREAM_RECOVERY_UNSUPPORTED"})
-  }
-
   status() {
     /** @type {Record<string, number>} */
     const streamStates = {}
@@ -280,23 +361,46 @@ export class ReverseGatewayConnection extends EventEmitter {
     if (failures.length > 0) throw new AggregateError(failures, "Reverse client spool cleanup failed")
   }
 
-  /** Orderly object detach is terminal; in-object physical reconnect uses paired resume. */
-  detach() { return this.close() }
+  /** Preserves durable streams while retiring this object for replacement. */
+  async detach() {
+    if (this.#closing) return
+    if (this.#options.connectionId === undefined) throw new Error("detach() requires an explicit stable connectionId")
+    if (this.#options.spoolDirectory === undefined) throw new Error("detach() requires an explicit persistent spoolDirectory")
+    this.#preserveSpools = true
+    this.#closing = true
+    if (this.#reconnectTimer !== undefined) clearTimeout(this.#reconnectTimer)
+    this.#reconnectTimer = undefined
+    this.#setState("closed")
+    const physical = this.#physical
+    physical?.pause()
+    physical?.destroy()
+    const frameQueue = this.#physicalFrameQueue
+    try {
+      await frameQueue
+      await Promise.all([...this.#streams.values()].map((stream) => this.#persistStreamState(stream)))
+    } finally {
+      physical?.destroy()
+      this.#physical = undefined
+      this.#rejectConnect?.(new SocketductSessionLostError("Reverse connection detached for durable recovery", "CONNECTION_DETACHED"))
+      this.#settleConnect()
+    }
+  }
 
   _writeStream(/** @type {number} */ streamId, /** @type {Buffer} */ data, /** @type {(error?: Error | null) => void} */ callback) {
     const stream = this.#streams.get(streamId)
-    if (stream === undefined || stream.localEnded || stream.state === "closed" || stream.state === "reset") { callback(new SocketductStreamError("Reverse stream is not writable", "ERR_STREAM_DESTROYED")); return }
+    if (this.#closing || stream === undefined || stream.localEnded || stream.state === "closed" || stream.state === "reset") { callback(new SocketductStreamError("Reverse stream is not writable", "ERR_STREAM_DESTROYED")); return }
     stream.pendingWrite = {data, offset: 0, callback}
     this.#progressPendingWrites()
   }
 
   _endStream(/** @type {number} */ streamId, /** @type {(error?: Error | null) => void} */ callback) {
     const stream = this.#streams.get(streamId)
-    if (stream === undefined || stream.localEnded) { callback(); return }
+    if (this.#closing || stream === undefined || stream.localEnded) { callback(); return }
     stream.localEnded = true
     if (stream.state === "closed") { callback(); return }
+    if (stream.opened && stream.state === "open") stream.state = "half-closed"
     stream.endCallback = callback
-    void this.#pumpStream(stream).catch((error) => this.#finishEnd(stream, asError(error)))
+    void this.#persistStreamState(stream).then(() => this.#pumpStream(stream)).catch((error) => this.#finishEnd(stream, asError(error)))
   }
 
   _destroyStream(/** @type {number} */ streamId, /** @type {Error | null} */ error) {
@@ -314,11 +418,26 @@ export class ReverseGatewayConnection extends EventEmitter {
   }
 
   _resumeReadable(/** @type {number} */ streamId) {
+    void streamId
+  }
+
+  _consumeReadable(/** @type {number} */ streamId, /** @type {number} */ bytes) {
     const stream = this.#streams.get(streamId)
-    if (stream?.pendingReceiveAck === undefined) return
-    const sequence = stream.pendingReceiveAck
-    stream.pendingReceiveAck = undefined
-    this.#ackReceived(stream, sequence)
+    if (stream === undefined || !Number.isSafeInteger(bytes) || bytes <= 0) return
+    let remaining = bytes
+    let acknowledgedSequence = stream.receiveAck
+    while (remaining > 0 && stream.receiveFrames.length > 0) {
+      const frame = /** @type {{sequence: number, remaining: number}} */ (stream.receiveFrames[0])
+      const consumed = Math.min(remaining, frame.remaining)
+      frame.remaining -= consumed
+      remaining -= consumed
+      if (frame.remaining === 0) {
+        acknowledgedSequence = frame.sequence
+        stream.receiveFrames.shift()
+      }
+    }
+    if (remaining !== 0) throw new Error("Reverse client readable accounting underflow")
+    this.#ackReceived(stream, acknowledgedSequence)
   }
 
   #allocateStreamId() {
@@ -361,6 +480,7 @@ export class ReverseGatewayConnection extends EventEmitter {
     physical.setKeepAlive(true, DEFAULT_KEEP_ALIVE_INITIAL_DELAY_MS)
     const decoder = new V3FrameDecoder({maxFrameBytes: Math.max(MAX_CONTROL_BYTES, this.#maxFrameBytes() + 4), maxBufferedBytes: Math.max(MAX_CONTROL_BYTES, this.#maxFrameBytes() + 4) * 2})
     let queue = Promise.resolve()
+    this.#physicalFrameQueue = queue
     let authenticated = false
     let selected = false
     /** @type {Error | undefined} */
@@ -368,14 +488,21 @@ export class ReverseGatewayConnection extends EventEmitter {
     const timer = setTimeout(() => physical.destroy(Object.assign(new Error("Reverse client TLS/auth timeout"), {code: "SOCKETDUCT_AUTH_TIMEOUT"})), this.#options.connectTimeoutMs ?? DEFAULT_CONNECT_TIMEOUT_MS)
     timer.unref?.()
     physical.once("secureConnect", () => {
-      const body = encodeV3Control({role: "client", token: this.#options.hub.token, versions: [3], connectionId: this.#connectionId, generation, compression: availableCodecs()})
-      void writeSocket(physical, encodeV3Frame({type: V3FrameType.AUTH, streamId: 0, sequence: generation}, body))
-        .catch((error) => physical.destroy(asError(error)))
+      // The attempted generation must survive a crash after AUTH reaches the
+      // hub but before AUTH_OK returns, otherwise a replacement could reuse a
+      // fenced generation after wall-clock rollback.
+      void Promise.all([...this.#streams.values()].map((stream) => this.#persistStreamState(stream))).then(async () => {
+        if (this.#physical !== physical || physical.destroyed || this.#closing) return
+        const body = encodeV3Control({role: "client", token: this.#options.hub.token, versions: [3], connectionId: this.#connectionId, generation, compression: availableCodecs()})
+        await writeSocket(physical, encodeV3Frame({type: V3FrameType.AUTH, streamId: 0, sequence: generation}, body))
+      }).catch((error) => physical.destroy(asError(error)))
     })
     physical.on("data", (chunk) => {
+      if (this.#closing || this.#physical !== physical) return
       physical.pause()
       queue = queue.then(async () => {
         for (const frame of decoder.push(chunk)) {
+          if (this.#closing || this.#physical !== physical) return
           if (frame.type === V3FrameType.AUTH_ERROR || frame.type === V3FrameType.GOAWAY) {
             const detail = decodeV3Control(frame.payload)
             throw connectionError(typeof detail.code === "string" ? detail.code : "CONNECTION_REJECTED")
@@ -387,6 +514,10 @@ export class ReverseGatewayConnection extends EventEmitter {
             authenticated = true
             this.#codec = ZSTD_CODEC
             this.#sendWindowLimit = Math.min(this.#initialWindow(), detail.initialWindow)
+            for (const stream of this.#streams.values()) {
+              stream.sendWindowLimit = Math.min(stream.sendWindowLimit, this.#sendWindowLimit)
+              stream.sendWindow = Math.min(stream.sendWindow, stream.sendWindowLimit)
+            }
             await writeSocket(physical, encodeV3Frame({type: V3FrameType.GATEWAY_SELECT, streamId: 0, sequence: generation}, encodeV3Control(this.#routeIdentityFields())))
             continue
           }
@@ -397,28 +528,26 @@ export class ReverseGatewayConnection extends EventEmitter {
             if (detail.generation !== generation || ("gatewayId" in selection ? detail.gatewayId !== selection.gatewayId || Object.hasOwn(detail, "routeClass") : detail.routeClass !== selection.routeClass || Object.hasOwn(detail, "gatewayId"))) throw fatalError("GATEWAY_SELECTION_FAILED")
             if (typeof detail.resumed !== "boolean") throw fatalError("GATEWAY_SELECTION_FAILED")
             selected = true
-            const reconnecting = this.#physicalConnects > 0
-            if (reconnecting && this.#streams.size > 0) {
-              if (!detail.resumed) {
-                for (const stream of [...this.#streams.values()]) stream.socket._receiveError(new SocketductSessionLostError("Reverse gateway hub lost the retained stream", "RESUME_SESSION_LOST"))
-                this.#completePhysical(generation, timer)
-              } else {
-                clearTimeout(timer)
-                this.#resuming = true
-                await this.#sendResumeManifest(generation)
-              }
+            if (detail.resumed) {
+              clearTimeout(timer)
+              this.#resuming = true
+              await this.#sendResumeManifest(generation)
+            } else if ([...this.#streams.values()].some((stream) => stream.openingSent || stream.opened)) {
+              for (const stream of [...this.#streams.values()]) stream.socket._receiveError(new SocketductSessionLostError("Reverse gateway hub lost the retained stream", "RESUME_SESSION_LOST"))
+              this.#completePhysical(generation, timer)
             } else {
               this.#completePhysical(generation, timer)
             }
             continue
           }
-          await this.#handleFrame(frame)
+          await this.#handleFrame(frame, physical)
         }
       }).catch((error) => {
         attemptError = asError(error)
         if (/** @type {{fatal?: unknown}} */ (attemptError).fatal === true) this.#fatal = true
         physical.destroy(attemptError)
       }).finally(() => { if (!physical.destroyed) physical.resume() })
+      this.#physicalFrameQueue = queue
     })
     physical.once("error", (error) => { attemptError = error; if (isFatalTransportError(error)) this.#fatal = true })
     physical.once("close", () => {
@@ -447,6 +576,8 @@ export class ReverseGatewayConnection extends EventEmitter {
         if (this.#closing) return
         if (this.#fatal) {
           const terminalError = attemptError ?? lost
+          const terminalCode = /** @type {{code?: unknown}} */ (terminalError).code
+          if (terminalCode === "STALE_GENERATION" || terminalCode === "CONNECTION_REPLACED") this.#preserveSpools = true
           this.#fatalError = terminalError
           this.#setState("closed", terminalError)
           logDiagnostic(this.#options.logger, "transport.failed", {
@@ -465,8 +596,9 @@ export class ReverseGatewayConnection extends EventEmitter {
     })
   }
 
-  /** @param {import('../protocol/v3-frame.js').V3Frame} frame - Hub frame. */
-  async #handleFrame(frame) {
+  /** @param {import('../protocol/v3-frame.js').V3Frame} frame - Hub frame. @param {import('node:tls').TLSSocket} physical - Owning physical generation. */
+  async #handleFrame(frame, physical) {
+    if (this.#closing || this.#physical !== physical) return
     if (frame.type === V3FrameType.PING) { await this.#sendControl(V3FrameType.PONG, 0, frame.sequence, {}); return }
     if (frame.type === V3FrameType.PONG) return
     if (frame.type === V3FrameType.RESUMED) {
@@ -476,7 +608,8 @@ export class ReverseGatewayConnection extends EventEmitter {
       this.#resumeNextPage += 1
       this.#resumeResults.push(...manifest.streams.filter((item) => typeof item === "object" && item !== null))
       if (manifest.final !== true) return
-      for (const item of this.#resumeResults) this.#applyResumeResult(item)
+      for (const item of this.#resumeResults) await this.#applyResumeResult(item)
+      if (this.#closing || this.#physical !== physical) return
       this.#resumeResults = []
       this.#resumeNextPage = 0
       this.#resuming = false
@@ -493,13 +626,16 @@ export class ReverseGatewayConnection extends EventEmitter {
       if (stream.state !== "opening" || frame.sequence !== 0 || detail.identifier !== stream.identifier || !validPeerWindow(detail.initialWindow)) throw new Error("Invalid reverse OPENED response")
       stream.sendWindowLimit = Math.min(stream.sendWindowLimit, detail.initialWindow)
       stream.sendWindow = Math.min(stream.sendWindow, stream.sendWindowLimit)
-      stream.state = "open"
+      stream.state = stream.localEnded ? "half-closed" : "open"
+      stream.opened = true
       logDiagnostic(this.#options.logger, "virtual_stream.opened", {
         streamId: stream.id,
         generation: this.#generation,
         ...this.#routeIdentityFields(),
         recovered: false
       })
+      await this.#persistStreamState(stream)
+      if (this.#closing || this.#physical !== physical) return
       stream.socket._markConnected()
       this.#progressPendingWrites()
       await this.#pumpStream(stream)
@@ -508,11 +644,11 @@ export class ReverseGatewayConnection extends EventEmitter {
     if (frame.type === V3FrameType.DATA) {
       if (stream.remoteEnded) throw new Error("Reverse client received DATA after END")
       const data = await decodeV3Data(frame, this.#codec, this.#options.compressionLimits)
+      if (this.#closing || this.#physical !== physical) return
       if (data.length === 0 || data.length > this.#maxFrameBytes() || frame.sequence !== stream.receiveSequence + data.length || frame.sequence - stream.receiveAck > this.#initialWindow()) throw new Error("Invalid reverse client DATA sequence or window")
       stream.receiveSequence = frame.sequence
-      const canContinue = stream.socket._receive(data)
-      if (canContinue) this.#ackReceived(stream, frame.sequence)
-      else stream.pendingReceiveAck = frame.sequence
+      stream.receiveFrames.push({sequence: frame.sequence, remaining: data.length})
+      stream.socket._receive(data)
       return
     }
     if (frame.type === V3FrameType.ACK) { this.#acknowledge(stream, frame.sequence); return }
@@ -520,12 +656,15 @@ export class ReverseGatewayConnection extends EventEmitter {
     if (frame.type === V3FrameType.END) {
       if (stream.remoteEnded || frame.sequence !== stream.receiveSequence) throw new Error("Invalid reverse client END")
       stream.remoteEnded = true
+      await this.#persistStreamState(stream)
+      if (this.#closing || this.#physical !== physical) return
       stream.socket._receiveEnd()
       return
     }
     if (frame.type === V3FrameType.CLOSE) {
       if (frame.sequence !== stream.receiveSequence) throw new Error("Invalid reverse client CLOSE")
       await this.#sendControl(V3FrameType.CLOSE, stream.id, stream.transmittedSequence, {})
+      if (this.#closing || this.#physical !== physical) return
       stream.state = "closed"
       logDiagnostic(this.#options.logger, "virtual_stream.closed", {
         streamId: stream.id,
@@ -558,7 +697,9 @@ export class ReverseGatewayConnection extends EventEmitter {
       identifier: stream.identifier,
       clientToGatewaySequence: stream.sendSequence,
       clientToGatewayAck: stream.sendAck,
-      gatewayToClientSequence: stream.receiveSequence,
+      // A replacement cannot inherit the dead VirtualSocket's process-only
+      // readable buffer. Ask the surviving gateway to replay above durable ACK.
+      gatewayToClientSequence: stream.recoveredObject ? stream.receiveAck : stream.receiveSequence,
       gatewayToClientAck: stream.receiveAck,
       localEnded: stream.localEnded,
       remoteEnded: stream.remoteEnded
@@ -576,7 +717,7 @@ export class ReverseGatewayConnection extends EventEmitter {
   }
 
   /** @param {unknown} value - Hub resume result. */
-  #applyResumeResult(value) {
+  async #applyResumeResult(value) {
     if (typeof value !== "object" || value === null) throw new Error("Invalid reverse client resume result")
     const result = /** @type {{streamId?: unknown, status?: unknown, clientToGatewaySequence?: unknown, clientToGatewayAck?: unknown, gatewayToClientSequence?: unknown, gatewayToClientAck?: unknown, clientEnded?: unknown, gatewayEnded?: unknown, opened?: unknown, code?: unknown}} */ (value)
     if (!Number.isInteger(result.streamId)) throw new Error("Invalid reverse client resume stream ID")
@@ -596,15 +737,20 @@ export class ReverseGatewayConnection extends EventEmitter {
     this.#acknowledge(stream, clientToGatewayAck)
     stream.transmittedSequence = clientToGatewayAck
     stream.receiveAck = gatewayToClientAck
-    stream.pendingReceiveAck = stream.pendingReceiveAck !== undefined && stream.pendingReceiveAck > stream.receiveAck ? stream.pendingReceiveAck : undefined
     stream.sendWindow = Math.max(0, stream.sendWindowLimit - (clientToGatewaySequence - clientToGatewayAck))
     for (const chunk of stream.replay) chunk.sent = chunk.sequence <= clientToGatewaySequence
     stream.endSent = /** @type {boolean} */ (result.clientEnded)
-    if (result.gatewayEnded === true && !stream.remoteEnded) {
+    if (result.gatewayEnded === true) {
+      stream.remoteEndPending = stream.recoveredObject || !stream.remoteEnded
       stream.remoteEnded = true
-      stream.socket._receiveEnd()
     }
-    if (result.opened === true && stream.state === "opening") stream.state = "open"
+    if (result.opened === true) {
+      stream.opened = true
+      if (stream.state === "opening") stream.state = stream.localEnded ? "half-closed" : "open"
+    }
+    stream.openingSent = true
+    stream.recoveredObject = false
+    await this.#persistStreamState(stream)
   }
 
   /** Marks one authenticated/selected physical generation ready. */
@@ -632,6 +778,10 @@ export class ReverseGatewayConnection extends EventEmitter {
     this.#settleConnect()
     for (const stream of this.#streams.values()) {
       if (stream.state !== "opening") stream.socket._markConnected()
+      if (stream.remoteEndPending) {
+        stream.remoteEndPending = false
+        stream.socket._receiveEnd()
+      }
       void this.#pumpStream(stream).catch((error) => stream.socket._receiveError(asError(error)))
     }
     this.#progressPendingWrites()
@@ -662,6 +812,7 @@ export class ReverseGatewayConnection extends EventEmitter {
           stream.endSent = false
           return
         }
+        await this.#persistStreamState(stream)
         this.#finishEnd(stream)
       }
     } finally { stream.pumping = false }
@@ -756,7 +907,7 @@ export class ReverseGatewayConnection extends EventEmitter {
     stream.replay = []
     stream.bufferedBytes = 0
     this.#finishWrite(stream, pendingWriteError)
-    this.#trackSpoolCleanup(stream)
+    if (!this.#preserveSpools) this.#trackSpoolCleanup(stream)
     this.#progressPendingWrites()
   }
 
@@ -841,19 +992,30 @@ export class ReverseGatewayConnection extends EventEmitter {
       version: 3,
       role: "reverse-client",
       connectionId: this.#connectionId,
+      generation: this.#generation,
+      ...this.#routeIdentityFields(),
       streamId: stream.id,
       identifier: stream.identifier,
       name: stream.name,
       target: stream.target,
+      state: stream.state,
+      opened: stream.opened,
+      openingSent: stream.openingSent,
       sendSequence: stream.sendSequence,
       sendAck: stream.sendAck,
       receiveSequence: stream.receiveSequence,
       receiveAck: stream.receiveAck,
       localEnded: stream.localEnded,
-      remoteEnded: stream.remoteEnded
+      remoteEnded: stream.remoteEnded,
+      endSent: stream.endSent
     }
   }
 
+  /** @param {ReverseClientStream} stream - Durable stream state. */
+  async #persistStreamState(stream) {
+    await this.#queueSpoolOperation(stream, () => this.#spool.writeRecoveryState(stream.identifier, this.#recoveryState(stream)))
+  }
+
   /** @param {ReverseClientStream} stream - Spool owner. @param {() => Promise<void>} operation - Serialized durable operation. */
   async #queueSpoolOperation(stream, operation) {
     const next = stream.spoolQueue.then(async () => {
@@ -880,7 +1042,10 @@ export class ReverseGatewayConnection extends EventEmitter {
 
   /** @param {ReverseClientStream} stream - Terminal stream. */
   #trackSpoolCleanup(stream) {
-    const task = this.#removeSpool(stream).catch((error) => {
+    const removal = stream.ownsSpool
+      ? this.#removeSpool(stream)
+      : stream.spoolReady.then(() => stream.ownsSpool ? this.#removeSpool(stream) : undefined)
+    const task = removal.catch((error) => {
       if (!stream.spoolCleanupFailureRecorded) {
         stream.spoolCleanupFailureRecorded = true
         this.#spoolCleanupFailures += 1
@@ -927,6 +1092,71 @@ function validateOptions(options) {
   if (options.initialGeneration !== undefined && (!Number.isSafeInteger(options.initialGeneration) || options.initialGeneration < 0)) throw new RangeError("initialGeneration must be a non-negative safe integer")
 }
 
+/**
+ * @param {Record<string, unknown>} value - Parsed durable state.
+ * @param {{connectionId: string, identifier: string, target: {host: string, port: number}, selection: {gatewayId: string} | {routeClass: string}, initialWindow: number, sessionName?: string}} expected - Caller-owned recovery identity.
+ * @returns {ReverseClientRecoveryState} Validated state.
+ */
+function validateRecoveryState(value, expected) {
+  const state = /** @type {Partial<ReverseClientRecoveryState>} */ (value)
+  const sequences = [state.generation, state.streamId, state.sendSequence, state.sendAck, state.receiveSequence, state.receiveAck]
+  const validSequences = sequences.every((sequence) => Number.isSafeInteger(sequence) && /** @type {number} */ (sequence) >= 0)
+  const validTarget = state.target?.host === expected.target.host && state.target?.port === expected.target.port
+  const expectedState = state.opened === true ? (state.localEnded === true ? "half-closed" : "open") : "opening"
+  const selectorMatches = "gatewayId" in expected.selection
+    ? state.gatewayId === expected.selection.gatewayId && !Object.hasOwn(state, "routeClass")
+    : state.routeClass === expected.selection.routeClass && !Object.hasOwn(state, "gatewayId")
+  if (
+    state.version !== 3
+    || state.role !== "reverse-client"
+    || state.connectionId !== expected.connectionId
+    || state.identifier !== expected.identifier
+    || !validSequences
+    || state.generation === 0
+    || state.generation === Number.MAX_SAFE_INTEGER
+    || state.streamId === 0
+    || /** @type {number} */ (state.streamId) % 2 === 0
+    || /** @type {number} */ (state.streamId) > 0xffff_fffd
+    || typeof state.name !== "string"
+    || state.name.length === 0
+    || state.name.length > 256
+    || !validTarget
+    || !selectorMatches
+    || typeof state.opened !== "boolean"
+    || state.openingSent !== true
+    || typeof state.localEnded !== "boolean"
+    || typeof state.remoteEnded !== "boolean"
+    || typeof state.endSent !== "boolean"
+    || (state.endSent && !state.localEnded)
+    || (state.endSent && state.opened !== true)
+    || (state.remoteEnded && state.opened !== true)
+    || (state.opened !== true && (/** @type {number} */ (state.sendSequence) > 0 || /** @type {number} */ (state.receiveSequence) > 0))
+    || state.state !== expectedState
+    || /** @type {number} */ (state.sendAck) > /** @type {number} */ (state.sendSequence)
+    || /** @type {number} */ (state.receiveAck) > /** @type {number} */ (state.receiveSequence)
+    || /** @type {number} */ (state.receiveSequence) - /** @type {number} */ (state.receiveAck) > expected.initialWindow
+    || (expected.sessionName !== undefined && state.name !== expected.sessionName)
+  ) throw new Error("Invalid reverse client recovery identity, selector, lifecycle, or sequence state")
+  return /** @type {ReverseClientRecoveryState} */ (state)
+}
+
+/**
+ * @param {ReverseClientRecoveryState} state - Validated durable state.
+ * @param {import('../spool/spool-file.js').SpoolChunk[]} chunks - Durable chunks.
+ * @param {number} maxBytes - Per-stream byte bound.
+ * @param {number} maxFrameBytes - Per-chunk frame bound.
+ */
+function validateRecoveredChunks(state, chunks, maxBytes, maxFrameBytes) {
+  let sequence = state.sendAck
+  let bytes = 0
+  for (const chunk of chunks) {
+    bytes += chunk.data.length
+    if (chunk.data.length === 0 || chunk.data.length > maxFrameBytes || !Number.isSafeInteger(sequence + chunk.data.length) || chunk.sequence !== sequence + chunk.data.length || chunk.sequence > state.sendSequence) throw new Error("Invalid reverse client recovery chunk sequence")
+    sequence = chunk.sequence
+  }
+  if (sequence !== state.sendSequence || bytes > maxBytes) throw new Error("Invalid or oversized reverse client recovery replay state")
+}
+
 /** @param {VirtualSocket} socket - Socket. @param {(error: Error | null, socket: VirtualSocket) => void} callback - Callback. */
 function installCallback(socket, callback) {
   let settled = false