socketduct 0.1.17 → 0.1.18

package/src/reverse/client.js

@@ -1,11 +1,14 @@
 /* eslint-disable jsdoc/escape-inline-tags, jsdoc/require-jsdoc, jsdoc/require-param, jsdoc/require-property-description, jsdoc/require-returns */
 import {randomBytes} from "node:crypto"
 import {EventEmitter} from "node:events"
+import {tmpdir} from "node:os"
+import {join} from "node:path"
 
 import {availableCodecs, getMaxDecompressedBytes, ZSTD_CODEC, zstdAvailable} from "../protocol/compression.js"
 import {errorDiagnostics, logDiagnostic} from "../logging/structured-logger.js"
 import {decodeV3Data, encodeV3Data, AdaptiveCompressor} from "../protocol/v3-compression.js"
 import {decodeV3Control, encodeV3Control, encodeV3Frame, V3FrameDecoder, V3FrameType} from "../protocol/v3-frame.js"
+import {SessionSpool} from "../spool/spool-file.js"
 import {SocketductSessionLostError, SocketductStreamError, VirtualSocket} from "../client/virtual-socket.js"
 import {
   asError,
@@ -27,6 +30,7 @@ import {
 
 const DEFAULT_STREAM_BUFFER_BYTES = 4 * 1024 * 1024
 const DEFAULT_CONNECTION_BUFFER_BYTES = 32 * 1024 * 1024
+const MANIFEST_PAGE_STREAMS = 64
 
 /**
  * @typedef {object} ReverseGatewayConnectionOptions
@@ -40,6 +44,8 @@ const DEFAULT_CONNECTION_BUFFER_BYTES = 32 * 1024 * 1024
  * @property {number} [maxStreams] - Concurrent stream limit.
  * @property {number} [maxStreamBufferBytes] - Per-stream client output bound.
  * @property {number} [maxConnectionBufferBytes] - Aggregate client output bound.
+ * @property {string} [spoolDirectory] - Durable replay spool root.
+ * @property {number} [maxAgeMs] - Maximum reverse-stream spool age.
  * @property {number} [connectTimeoutMs] - TLS/auth deadline.
  * @property {number} [reconnectDelayMs] - Initial reconnect delay.
  * @property {number} [maxReconnectDelayMs] - Maximum reconnect delay.
@@ -82,6 +88,11 @@ const DEFAULT_CONNECTION_BUFFER_BYTES = 32 * 1024 * 1024
  * @property {boolean} localEnded
  * @property {boolean} endSent
  * @property {((error?: Error | null) => void) | undefined} endCallback
+ * @property {Promise<void>} spoolReady
+ * @property {Promise<void>} spoolQueue
+ * @property {number} spoolReservedBytes
+ * @property {Promise<void> | undefined} spoolRemovalPromise
+ * @property {boolean} spoolCleanupFailureRecorded
  */
 
 /** Server3/TensorBuzz-facing virtual socket connection to an exact gateway or server-resolved route class. */
@@ -99,6 +110,9 @@ export class ReverseGatewayConnection extends EventEmitter {
   /** @type {Error | undefined} */ #fatalError
   /** @type {boolean} */ #referenced = true
   /** @type {boolean} */ #warm = false
+  /** @type {boolean} */ #resuming = false
+  /** @type {Array<Record<string, unknown>>} */ #resumeResults = []
+  /** @type {number} */ #resumeNextPage = 0
   /** @type {ReturnType<typeof setTimeout> | undefined} */ #reconnectTimer
   /** @type {Promise<void> | undefined} */ #connectPromise
   /** @type {(() => void) | undefined} */ #resolveConnect
@@ -112,6 +126,11 @@ export class ReverseGatewayConnection extends EventEmitter {
   /** @type {number} */ #rawDataBytes = 0
   /** @type {number} */ #wireDataBytes = 0
   /** @type {number} */ #lastProgressedStreamId = 0
+  /** @type {Promise<void>} */ #openQueue = Promise.resolve()
+  /** @type {SessionSpool} */ #spool
+  /** @type {number} */ #spoolBytes = 0
+  /** @type {number} */ #spoolCleanupFailures = 0
+  /** @type {Set<Promise<void>>} */ #spoolCleanupTasks = new Set()
 
   /** @param {ReverseGatewayConnectionOptions} options - Hub route and safety policy. */
   constructor(options) {
@@ -119,6 +138,11 @@ export class ReverseGatewayConnection extends EventEmitter {
     validateOptions(options)
     this.#options = options
     this.#connectionId = options.connectionId ?? randomBytes(24).toString("base64url")
+    this.#spool = new SessionSpool({
+      rootDir: options.spoolDirectory ?? join(tmpdir(), `socketduct-${typeof process.getuid === "function" ? process.getuid() : "user"}`, "reverse-client", this.#connectionId),
+      maxBytes: options.maxStreamBufferBytes ?? DEFAULT_STREAM_BUFFER_BYTES,
+      maxAgeMs: options.maxAgeMs ?? 24 * 60 * 60 * 1000
+    })
     this.#generation = options.initialGeneration ?? Date.now() * 1000
     this.#reconnectDelay = options.reconnectDelayMs ?? 250
     this.#sendWindowLimit = this.#initialWindow()
@@ -176,9 +200,17 @@ export class ReverseGatewayConnection extends EventEmitter {
       progressingWrite: false,
       localEnded: false,
       endSent: false,
-      endCallback: undefined
+      endCallback: undefined,
+      spoolReady: Promise.resolve(),
+      spoolQueue: Promise.resolve(),
+      spoolReservedBytes: 0,
+      spoolRemovalPromise: undefined,
+      spoolCleanupFailureRecorded: false
     }
     this.#streams.set(id, stream)
+    stream.spoolReady = this.#spool.createNew({sessionId: identifier, ownerId: "reverse-client-v3"})
+      .then(() => this.#spool.writeRecoveryState(identifier, this.#recoveryState(stream)))
+    stream.spoolQueue = stream.spoolReady.catch(() => {})
     logDiagnostic(this.#options.logger, "virtual_stream.opening", {
       streamId: id,
       generation: this.#generation,
@@ -186,13 +218,17 @@ export class ReverseGatewayConnection extends EventEmitter {
       recovered: false
     })
     if (callback !== undefined) installCallback(socket, callback)
-    void this.#sendControl(V3FrameType.OPEN, id, 0, {...this.#routeIdentityFields(), identifier, name, target, initialWindow: this.#initialWindow()}).catch((error) => socket._receiveError(asError(error)))
+    const opening = this.#openQueue
+      .then(() => stream.spoolReady)
+      .then(() => this.#streams.get(id) === stream ? this.#sendControl(V3FrameType.OPEN, id, 0, {...this.#routeIdentityFields(), identifier, name, target, initialWindow: this.#initialWindow()}) : undefined)
+    this.#openQueue = opening.then(() => undefined, () => undefined)
+    void opening.catch((error) => socket._receiveError(asError(error)))
     return socket
   }
 
-  /** Reverse established operations are intentionally not recoverable across process/transport loss. */
+  /** Process replacement cannot inherit the live reverse target/app socket pair. */
   registerRecoveredConnection() {
-    throw Object.assign(new Error("Reverse gateway streams cannot be recovered; reconcile the Docker operation before retrying"), {code: "REVERSE_STREAM_RECOVERY_UNSUPPORTED"})
+    throw Object.assign(new Error("Reverse gateway process recovery is unsupported; live streams resume only within the original process"), {code: "REVERSE_STREAM_RECOVERY_UNSUPPORTED"})
   }
 
   status() {
@@ -207,8 +243,8 @@ export class ReverseGatewayConnection extends EventEmitter {
       queuedBytes: this.#bufferedBytes,
       dataQueuedBytes: this.#bufferedBytes,
       controlQueuedBytes: 0,
-      spoolBytes: 0,
-      spoolCleanupFailures: 0,
+      spoolBytes: this.#spoolBytes,
+      spoolCleanupFailures: this.#spoolCleanupFailures,
       physicalConnects: this.#physicalConnects,
       reconnectAttempts: this.#reconnectAttempts,
       reconnectSuccesses: Math.max(0, this.#physicalConnects - 1),
@@ -239,9 +275,12 @@ export class ReverseGatewayConnection extends EventEmitter {
     }
     this.#rejectConnect?.(closedError())
     this.#settleConnect()
+    const cleanup = await Promise.allSettled([...this.#spoolCleanupTasks])
+    const failures = cleanup.filter((result) => result.status === "rejected").map((result) => asError(result.reason))
+    if (failures.length > 0) throw new AggregateError(failures, "Reverse client spool cleanup failed")
   }
 
-  /** Detach is terminal because ambiguous reverse streams are never durably replayed. */
+  /** Orderly object detach is terminal; in-object physical reconnect uses paired resume. */
   detach() { return this.close() }
 
   _writeStream(/** @type {number} */ streamId, /** @type {Buffer} */ data, /** @type {(error?: Error | null) => void} */ callback) {
@@ -356,28 +395,21 @@ export class ReverseGatewayConnection extends EventEmitter {
             const detail = decodeV3Control(frame.payload)
             const selection = this.#routeIdentityFields()
             if (detail.generation !== generation || ("gatewayId" in selection ? detail.gatewayId !== selection.gatewayId || Object.hasOwn(detail, "routeClass") : detail.routeClass !== selection.routeClass || Object.hasOwn(detail, "gatewayId"))) throw fatalError("GATEWAY_SELECTION_FAILED")
+            if (typeof detail.resumed !== "boolean") throw fatalError("GATEWAY_SELECTION_FAILED")
             selected = true
-            clearTimeout(timer)
-            this.#physicalConnects += 1
-            this.#reconnectDelay = this.#options.reconnectDelayMs ?? 250
-            this.#setState("ready")
-            logDiagnostic(this.#options.logger, "transport.connected", {
-              generation,
-              attempt: this.#physicalAttempts,
-              ...this.#routeIdentityFields(),
-              physicalConnects: this.#physicalConnects,
-              reconnect: this.#physicalConnects > 1
-            })
-            if (this.#physicalConnects > 1) {
-              logDiagnostic(this.#options.logger, "transport.reconnect_succeeded", {
-                generation,
-                attempt: this.#reconnectAttempts,
-                ...this.#routeIdentityFields(),
-                physicalConnects: this.#physicalConnects
-              })
+            const reconnecting = this.#physicalConnects > 0
+            if (reconnecting && this.#streams.size > 0) {
+              if (!detail.resumed) {
+                for (const stream of [...this.#streams.values()]) stream.socket._receiveError(new SocketductSessionLostError("Reverse gateway hub lost the retained stream", "RESUME_SESSION_LOST"))
+                this.#completePhysical(generation, timer)
+              } else {
+                clearTimeout(timer)
+                this.#resuming = true
+                await this.#sendResumeManifest(generation)
+              }
+            } else {
+              this.#completePhysical(generation, timer)
             }
-            this.#resolveConnect?.()
-            this.#settleConnect()
             continue
           }
           await this.#handleFrame(frame)
@@ -395,8 +427,11 @@ export class ReverseGatewayConnection extends EventEmitter {
         if (this.#physical !== physical) return
         this.#physical = undefined
         this.#codec = undefined
+        this.#resuming = false
+        this.#resumeResults = []
+        this.#resumeNextPage = 0
         if (!this.#closing && !this.#fatal) this.#setState("reconnecting")
-        const lost = new SocketductSessionLostError("Reverse gateway transport was lost; operation outcome may be uncertain", "GATEWAY_DISCONNECTED")
+        const lost = new SocketductSessionLostError("Reverse gateway physical transport was interrupted", "GATEWAY_DISCONNECTED")
         if (!this.#closing) {
           logDiagnostic(this.#options.logger, "transport.disconnected", {
             generation,
@@ -405,7 +440,10 @@ export class ReverseGatewayConnection extends EventEmitter {
             ...errorDiagnostics(attemptError ?? lost, "GATEWAY_DISCONNECTED")
           })
         }
-        for (const stream of [...this.#streams.values()]) stream.socket._receiveError(lost)
+        for (const stream of this.#streams.values()) {
+          stream.sendWindow = stream.sendWindowLimit
+          for (const replay of stream.replay) replay.sent = false
+        }
         if (this.#closing) return
         if (this.#fatal) {
           const terminalError = attemptError ?? lost
@@ -419,6 +457,7 @@ export class ReverseGatewayConnection extends EventEmitter {
           })
           this.#rejectConnect?.(terminalError)
           this.#settleConnect()
+          for (const stream of [...this.#streams.values()]) stream.socket._receiveError(terminalError)
           return
         }
         if (this.#warm || this.#options.continuousReconnect !== false) this.#scheduleReconnect(attemptError)
@@ -430,6 +469,20 @@ export class ReverseGatewayConnection extends EventEmitter {
   async #handleFrame(frame) {
     if (frame.type === V3FrameType.PING) { await this.#sendControl(V3FrameType.PONG, 0, frame.sequence, {}); return }
     if (frame.type === V3FrameType.PONG) return
+    if (frame.type === V3FrameType.RESUMED) {
+      if (!this.#resuming) throw new Error("Unexpected reverse client RESUMED response")
+      const manifest = decodeV3Control(frame.payload)
+      if (manifest.page !== this.#resumeNextPage || !Array.isArray(manifest.streams)) throw new Error("Invalid reverse client RESUMED response")
+      this.#resumeNextPage += 1
+      this.#resumeResults.push(...manifest.streams.filter((item) => typeof item === "object" && item !== null))
+      if (manifest.final !== true) return
+      for (const item of this.#resumeResults) this.#applyResumeResult(item)
+      this.#resumeResults = []
+      this.#resumeNextPage = 0
+      this.#resuming = false
+      this.#completePhysical(this.#generation)
+      return
+    }
     const stream = this.#streams.get(frame.streamId)
     if (stream === undefined) {
       if (frame.type !== V3FrameType.CLOSE && frame.type !== V3FrameType.RESET) await this.#sendControl(V3FrameType.RESET, frame.streamId, 0, {code: "UNKNOWN_STREAM"})
@@ -496,6 +549,94 @@ export class ReverseGatewayConnection extends EventEmitter {
     throw new Error("Unsupported reverse client frame")
   }
 
+  /** Sends the client half of the paired reverse-stream resume manifest. */
+  async #sendResumeManifest(/** @type {number} */ generation) {
+    const activeStreams = [...this.#streams.values()]
+    await Promise.all(activeStreams.map((stream) => this.#queueSpoolOperation(stream, () => this.#spool.writeRecoveryState(stream.identifier, this.#recoveryState(stream)))))
+    const streams = activeStreams.filter((stream) => this.#streams.get(stream.id) === stream).map((stream) => ({
+      streamId: stream.id,
+      identifier: stream.identifier,
+      clientToGatewaySequence: stream.sendSequence,
+      clientToGatewayAck: stream.sendAck,
+      gatewayToClientSequence: stream.receiveSequence,
+      gatewayToClientAck: stream.receiveAck,
+      localEnded: stream.localEnded,
+      remoteEnded: stream.remoteEnded
+    }))
+    this.#resumeResults = []
+    this.#resumeNextPage = 0
+    const pageCount = Math.max(1, Math.ceil(streams.length / MANIFEST_PAGE_STREAMS))
+    for (let page = 0; page < pageCount; page += 1) {
+      await this.#sendControl(V3FrameType.RESUME, 0, generation, {
+        page,
+        final: page === pageCount - 1,
+        streams: streams.slice(page * MANIFEST_PAGE_STREAMS, (page + 1) * MANIFEST_PAGE_STREAMS)
+      })
+    }
+  }
+
+  /** @param {unknown} value - Hub resume result. */
+  #applyResumeResult(value) {
+    if (typeof value !== "object" || value === null) throw new Error("Invalid reverse client resume result")
+    const result = /** @type {{streamId?: unknown, status?: unknown, clientToGatewaySequence?: unknown, clientToGatewayAck?: unknown, gatewayToClientSequence?: unknown, gatewayToClientAck?: unknown, clientEnded?: unknown, gatewayEnded?: unknown, opened?: unknown, code?: unknown}} */ (value)
+    if (!Number.isInteger(result.streamId)) throw new Error("Invalid reverse client resume stream ID")
+    const stream = this.#streams.get(/** @type {number} */ (result.streamId))
+    if (stream === undefined) return
+    if (result.status !== "resumed") {
+      const code = typeof result.code === "string" ? result.code : "RESUME_REJECTED"
+      stream.socket._receiveError(new SocketductSessionLostError("Reverse gateway stream could not be resumed", code))
+      return
+    }
+    const clientToGatewaySequence = /** @type {number} */ (result.clientToGatewaySequence)
+    const clientToGatewayAck = /** @type {number} */ (result.clientToGatewayAck)
+    const gatewayToClientSequence = /** @type {number} */ (result.gatewayToClientSequence)
+    const gatewayToClientAck = /** @type {number} */ (result.gatewayToClientAck)
+    const validClientToGatewayBoundary = clientToGatewaySequence === stream.sendAck || stream.replay.some((chunk) => chunk.sequence === clientToGatewaySequence)
+    if (!Number.isSafeInteger(clientToGatewaySequence) || clientToGatewaySequence < stream.sendAck || clientToGatewaySequence > stream.sendSequence || !Number.isSafeInteger(clientToGatewayAck) || clientToGatewayAck < stream.sendAck || clientToGatewayAck > clientToGatewaySequence || !validClientToGatewayBoundary || !Number.isSafeInteger(gatewayToClientSequence) || gatewayToClientSequence !== stream.receiveSequence || !Number.isSafeInteger(gatewayToClientAck) || gatewayToClientAck < stream.receiveAck || gatewayToClientAck > gatewayToClientSequence || typeof result.clientEnded !== "boolean" || typeof result.gatewayEnded !== "boolean" || typeof result.opened !== "boolean") throw new Error("Invalid reverse client resume positions")
+    this.#acknowledge(stream, clientToGatewayAck)
+    stream.transmittedSequence = clientToGatewayAck
+    stream.receiveAck = gatewayToClientAck
+    stream.pendingReceiveAck = stream.pendingReceiveAck !== undefined && stream.pendingReceiveAck > stream.receiveAck ? stream.pendingReceiveAck : undefined
+    stream.sendWindow = Math.max(0, stream.sendWindowLimit - (clientToGatewaySequence - clientToGatewayAck))
+    for (const chunk of stream.replay) chunk.sent = chunk.sequence <= clientToGatewaySequence
+    stream.endSent = /** @type {boolean} */ (result.clientEnded)
+    if (result.gatewayEnded === true && !stream.remoteEnded) {
+      stream.remoteEnded = true
+      stream.socket._receiveEnd()
+    }
+    if (result.opened === true && stream.state === "opening") stream.state = "open"
+  }
+
+  /** Marks one authenticated/selected physical generation ready. */
+  #completePhysical(/** @type {number} */ generation, /** @type {ReturnType<typeof setTimeout> | undefined} */ timer = undefined) {
+    if (timer !== undefined) clearTimeout(timer)
+    this.#physicalConnects += 1
+    this.#reconnectDelay = this.#options.reconnectDelayMs ?? 250
+    this.#setState("ready")
+    logDiagnostic(this.#options.logger, "transport.connected", {
+      generation,
+      attempt: this.#physicalAttempts,
+      ...this.#routeIdentityFields(),
+      physicalConnects: this.#physicalConnects,
+      reconnect: this.#physicalConnects > 1
+    })
+    if (this.#physicalConnects > 1) {
+      logDiagnostic(this.#options.logger, "transport.reconnect_succeeded", {
+        generation,
+        attempt: this.#reconnectAttempts,
+        ...this.#routeIdentityFields(),
+        physicalConnects: this.#physicalConnects
+      })
+    }
+    this.#resolveConnect?.()
+    this.#settleConnect()
+    for (const stream of this.#streams.values()) {
+      if (stream.state !== "opening") stream.socket._markConnected()
+      void this.#pumpStream(stream).catch((error) => stream.socket._receiveError(asError(error)))
+    }
+    this.#progressPendingWrites()
+  }
+
   /** @param {ReverseClientStream} stream - Stream. */
   async #pumpStream(stream) {
     if (stream.pumping || stream.state === "opening" || !this.#streams.has(stream.id) || this.#state !== "ready") return
@@ -516,7 +657,11 @@ export class ReverseGatewayConnection extends EventEmitter {
       if (stream.localEnded && !stream.endSent && stream.replay.every((chunk) => chunk.sent)) {
         stream.endSent = true
         stream.state = "half-closed"
-        await this.#sendControl(V3FrameType.END, stream.id, stream.sendSequence, {})
+        const delivered = await this.#sendControl(V3FrameType.END, stream.id, stream.sendSequence, {})
+        if (!delivered) {
+          stream.endSent = false
+          return
+        }
         this.#finishEnd(stream)
       }
     } finally { stream.pumping = false }
@@ -533,8 +678,7 @@ export class ReverseGatewayConnection extends EventEmitter {
         if (available <= 0) return
         const chunk = Buffer.from(pending.data.subarray(pending.offset, Math.min(pending.data.length, pending.offset + available)))
         try {
-          if (this.#options.aggregateBudget?.reserveMemory !== undefined) this.#options.aggregateBudget.reserveMemory(chunk.length)
-          else this.#options.aggregateBudget?.reserveData(chunk.length)
+          this.#options.aggregateBudget?.reserveData(chunk.length)
         } catch (error) {
           this.#finishWrite(stream, asError(error))
           return
@@ -544,11 +688,19 @@ export class ReverseGatewayConnection extends EventEmitter {
         stream.replay.push({sequence: stream.sendSequence, data: chunk, sent: false})
         stream.bufferedBytes += chunk.length
         this.#bufferedBytes += chunk.length
+        stream.spoolReservedBytes += chunk.length
+        this.#spoolBytes += chunk.length
+        await this.#queueSpoolOperation(stream, async () => {
+          await this.#spool.appendChunk({sessionId: stream.identifier, ownerId: "reverse-client-v3", sequence: stream.sendSequence, data: chunk})
+          await this.#spool.writeRecoveryState(stream.identifier, this.#recoveryState(stream))
+        })
       }
       await this.#pumpStream(stream)
       if (pending.offset === pending.data.length && stream.pendingWrite === pending) this.#finishWrite(stream)
     } catch (error) {
-      this.#finishWrite(stream, asError(error))
+      const failure = asError(error)
+      this.#finishWrite(stream, failure)
+      stream.socket._receiveError(failure)
     } finally {
       stream.progressingWrite = false
       this.#progressPendingWrites()
@@ -601,10 +753,10 @@ export class ReverseGatewayConnection extends EventEmitter {
     if (!this.#streams.delete(stream.id)) return
     this.#bufferedBytes = Math.max(0, this.#bufferedBytes - stream.bufferedBytes)
     this.#options.aggregateBudget?.releaseMemory(stream.bufferedBytes)
-    if (this.#options.aggregateBudget?.reserveMemory === undefined) this.#options.aggregateBudget?.releaseSpool(stream.bufferedBytes)
     stream.replay = []
     stream.bufferedBytes = 0
     this.#finishWrite(stream, pendingWriteError)
+    this.#trackSpoolCleanup(stream)
     this.#progressPendingWrites()
   }
 
@@ -619,8 +771,15 @@ export class ReverseGatewayConnection extends EventEmitter {
     stream.bufferedBytes = Math.max(0, stream.bufferedBytes - released)
     this.#bufferedBytes = Math.max(0, this.#bufferedBytes - released)
     this.#options.aggregateBudget?.releaseMemory(released)
-    if (this.#options.aggregateBudget?.reserveMemory === undefined) this.#options.aggregateBudget?.releaseSpool(released)
     stream.sendWindow = Math.min(stream.sendWindowLimit, stream.sendWindow + delta)
+    void this.#queueSpoolOperation(stream, async () => {
+      await this.#spool.writeRecoveryState(stream.identifier, this.#recoveryState(stream))
+      await this.#spool.truncateAcked(stream.identifier, sequence)
+      if (released > stream.spoolReservedBytes) throw new Error("Reverse client spool budget release underflow")
+      stream.spoolReservedBytes -= released
+      this.#spoolBytes = Math.max(0, this.#spoolBytes - released)
+      this.#options.aggregateBudget?.releaseSpool(released)
+    }).catch((error) => stream.socket._receiveError(asError(error)))
     this.#progressPendingWrites()
     void this.#pumpStream(stream).catch((error) => stream.socket._receiveError(asError(error)))
   }
@@ -636,7 +795,9 @@ export class ReverseGatewayConnection extends EventEmitter {
   #ackReceived(stream, sequence) {
     if (sequence <= stream.receiveAck) return
     stream.receiveAck = sequence
-    void this.#sendControl(V3FrameType.ACK, stream.id, sequence, {}).catch(() => {})
+    void this.#queueSpoolOperation(stream, () => this.#spool.writeRecoveryState(stream.identifier, this.#recoveryState(stream)))
+      .then(() => this.#sendControl(V3FrameType.ACK, stream.id, sequence, {}).catch(() => {}))
+      .catch((error) => stream.socket._receiveError(asError(error)))
   }
 
   /** @param {number} type - Type. @param {number} streamId - Stream. @param {number} sequence - Sequence. @param {Record<string, unknown>} body - Body. */
@@ -674,6 +835,62 @@ export class ReverseGatewayConnection extends EventEmitter {
     }
   }
 
+  /** @param {ReverseClientStream} stream - Durable stream state. */
+  #recoveryState(stream) {
+    return {
+      version: 3,
+      role: "reverse-client",
+      connectionId: this.#connectionId,
+      streamId: stream.id,
+      identifier: stream.identifier,
+      name: stream.name,
+      target: stream.target,
+      sendSequence: stream.sendSequence,
+      sendAck: stream.sendAck,
+      receiveSequence: stream.receiveSequence,
+      receiveAck: stream.receiveAck,
+      localEnded: stream.localEnded,
+      remoteEnded: stream.remoteEnded
+    }
+  }
+
+  /** @param {ReverseClientStream} stream - Spool owner. @param {() => Promise<void>} operation - Serialized durable operation. */
+  async #queueSpoolOperation(stream, operation) {
+    const next = stream.spoolQueue.then(async () => {
+      await stream.spoolReady
+      await operation()
+    })
+    stream.spoolQueue = next.catch(() => {})
+    await next
+  }
+
+  /** @param {ReverseClientStream} stream - Terminal stream. */
+  async #removeSpool(stream) {
+    if (stream.spoolRemovalPromise === undefined) {
+      stream.spoolRemovalPromise = this.#queueSpoolOperation(stream, async () => {
+        await this.#spool.remove(stream.identifier)
+        const removed = stream.spoolReservedBytes
+        stream.spoolReservedBytes = 0
+        this.#spoolBytes = Math.max(0, this.#spoolBytes - removed)
+        this.#options.aggregateBudget?.releaseSpool(removed)
+      })
+    }
+    await stream.spoolRemovalPromise
+  }
+
+  /** @param {ReverseClientStream} stream - Terminal stream. */
+  #trackSpoolCleanup(stream) {
+    const task = this.#removeSpool(stream).catch((error) => {
+      if (!stream.spoolCleanupFailureRecorded) {
+        stream.spoolCleanupFailureRecorded = true
+        this.#spoolCleanupFailures += 1
+      }
+      throw error
+    })
+    this.#spoolCleanupTasks.add(task)
+    void task.catch(() => {}).finally(() => this.#spoolCleanupTasks.delete(task))
+  }
+
   #settleConnect() { this.#connectPromise = undefined; this.#resolveConnect = undefined; this.#rejectConnect = undefined }
   #setState(/** @type {'configured' | 'connecting' | 'ready' | 'reconnecting' | 'closed'} */ state, /** @type {Error | undefined} */ error = undefined) { if (this.#state === state) return; this.#state = state; this.emit("stateChange", state, error) }
   /** @returns {{gatewayId: string} | {routeClass: string}} Stable physical route identity. */
@@ -705,6 +922,8 @@ function validateOptions(options) {
   positiveInteger(options.maxStreams, "maxStreams", DEFAULT_MAX_STREAMS)
   positiveInteger(options.maxStreamBufferBytes, "maxStreamBufferBytes", DEFAULT_STREAM_BUFFER_BYTES)
   positiveInteger(options.maxConnectionBufferBytes, "maxConnectionBufferBytes", DEFAULT_CONNECTION_BUFFER_BYTES)
+  if (options.spoolDirectory !== undefined && (typeof options.spoolDirectory !== "string" || options.spoolDirectory.length === 0)) throw new TypeError("spoolDirectory must be a non-empty string")
+  positiveInteger(options.maxAgeMs, "maxAgeMs", 24 * 60 * 60 * 1000)
   if (options.initialGeneration !== undefined && (!Number.isSafeInteger(options.initialGeneration) || options.initialGeneration < 0)) throw new RangeError("initialGeneration must be a non-negative safe integer")
 }