socketduct 0.1.17 → 0.1.19

package/src/reverse/gateway.js

@@ -1,11 +1,14 @@
 /* eslint-disable jsdoc/escape-inline-tags, jsdoc/require-jsdoc, jsdoc/require-param, jsdoc/require-property-description, jsdoc/require-returns */
 import {EventEmitter} from "node:events"
 import {createConnection as netConnect} from "node:net"
+import {tmpdir} from "node:os"
+import {join} from "node:path"
 
 import {availableCodecs, getMaxDecompressedBytes, ZSTD_CODEC, zstdAvailable} from "../protocol/compression.js"
 import {errorDiagnostics, logDiagnostic} from "../logging/structured-logger.js"
 import {decodeV3Data, encodeV3Data, AdaptiveCompressor} from "../protocol/v3-compression.js"
 import {decodeV3Control, encodeV3Control, encodeV3Frame, V3FrameDecoder, V3FrameType} from "../protocol/v3-frame.js"
+import {SessionSpool} from "../spool/spool-file.js"
 import {
   asError,
   createHubSocket,
@@ -25,6 +28,7 @@ import {
 
 const DEFAULT_STREAM_BUFFER_BYTES = 4 * 1024 * 1024
 const DEFAULT_TOTAL_BUFFER_BYTES = 32 * 1024 * 1024
+const MANIFEST_PAGE_STREAMS = 64
 
 /**
  * @typedef {object} OutboundReverseGatewayOptions
@@ -39,6 +43,8 @@ const DEFAULT_TOTAL_BUFFER_BYTES = 32 * 1024 * 1024
  * @property {number} [maxStreams] - Concurrent local target bound.
  * @property {number} [maxStreamBufferBytes] - Per-stream unacknowledged target output bound.
  * @property {number} [maxTotalBufferBytes] - Gateway-wide unacknowledged target output bound.
+ * @property {string} [spoolDirectory] - Durable target-output replay spool root.
+ * @property {number} [maxAgeMs] - Maximum reverse-stream spool age.
  * @property {number} [connectTimeoutMs] - TLS/auth/register deadline.
  * @property {number} [targetConnectTimeoutMs] - Local target connection deadline.
  * @property {number} [reconnectDelayMs] - Initial reconnect delay.
@@ -56,7 +62,7 @@ const DEFAULT_TOTAL_BUFFER_BYTES = 32 * 1024 * 1024
  * @property {string} identifier
  * @property {{host: string, port: number}} target
  * @property {import('node:net').Socket} targetSocket
- * @property {'opening' | 'open' | 'half-closed'} state
+ * @property {'opening' | 'open' | 'half-closed' | 'closing'} state
  * @property {number} receiveSequence
  * @property {number} receiveAck
  * @property {boolean} remoteEnded
@@ -73,9 +79,16 @@ const DEFAULT_TOTAL_BUFFER_BYTES = 32 * 1024 * 1024
  * @property {boolean} targetClosed
  * @property {string | undefined} targetResetCode
  * @property {boolean} endSent
+ * @property {boolean} closeSent
  * @property {ReturnType<typeof setTimeout> | undefined} openTimer
  * @property {boolean} targetOpened
  * @property {boolean} targetTerminalLogged
+ * @property {Promise<void>} openedReady
+ * @property {Promise<void>} spoolReady
+ * @property {Promise<void>} spoolQueue
+ * @property {number} spoolReservedBytes
+ * @property {Promise<void> | undefined} spoolRemovalPromise
+ * @property {boolean} spoolCleanupFailureRecorded
  */
 
 /** One outbound home member carrying many hub-opened local target streams. */
@@ -90,6 +103,9 @@ export class OutboundReverseGateway extends EventEmitter {
   /** @type {boolean} */ #closing = false
   /** @type {boolean} */ #fatal = false
   /** @type {boolean} */ #referenced = true
+  /** @type {boolean} */ #resuming = false
+  /** @type {Array<Record<string, unknown>>} */ #resumeResults = []
+  /** @type {number} */ #resumeNextPage = 0
   /** @type {ReturnType<typeof setTimeout> | undefined} */ #reconnectTimer
   /** @type {Promise<void> | undefined} */ #startPromise
   /** @type {(() => void) | undefined} */ #resolveStart
@@ -104,12 +120,21 @@ export class OutboundReverseGateway extends EventEmitter {
   /** @type {number} */ #bufferedBytes = 0
   /** @type {number} */ #rawDataBytes = 0
   /** @type {number} */ #wireDataBytes = 0
+  /** @type {SessionSpool} */ #spool
+  /** @type {number} */ #spoolBytes = 0
+  /** @type {number} */ #spoolCleanupFailures = 0
+  /** @type {Set<Promise<void>>} */ #spoolCleanupTasks = new Set()
 
   /** @param {OutboundReverseGatewayOptions} options - Identity, hub, TLS and local policy. */
   constructor(options) {
     super()
     validateOptions(options)
     this.#options = options
+    this.#spool = new SessionSpool({
+      rootDir: options.spoolDirectory ?? join(tmpdir(), `socketduct-${typeof process.getuid === "function" ? process.getuid() : "user"}`, "reverse-gateway", `${options.gatewayId}-${options.connectionId}`),
+      maxBytes: options.maxStreamBufferBytes ?? DEFAULT_STREAM_BUFFER_BYTES,
+      maxAgeMs: options.maxAgeMs ?? 24 * 60 * 60 * 1000
+    })
     this.#generation = options.initialGeneration ?? Date.now() * 1000
     this.#reconnectDelay = options.reconnectDelayMs ?? 250
     this.#sendWindowLimit = this.#initialWindow()
@@ -140,8 +165,8 @@ export class OutboundReverseGateway extends EventEmitter {
       targetOpens: this.#targetOpens,
       targetOpenFailures: this.#targetOpenFailures,
       queuedBytes: this.#bufferedBytes,
-      spoolBytes: 0,
-      spoolCleanupFailures: 0,
+      spoolBytes: this.#spoolBytes,
+      spoolCleanupFailures: this.#spoolCleanupFailures,
       compression: this.#codec ?? null,
       rawDataBytes: this.#rawDataBytes,
       wireDataBytes: this.#wireDataBytes,
@@ -173,6 +198,9 @@ export class OutboundReverseGateway extends EventEmitter {
     }
     this.#rejectStart?.(closedError())
     this.#settleStart()
+    const cleanup = await Promise.allSettled([...this.#spoolCleanupTasks])
+    const failures = cleanup.filter((result) => result.status === "rejected").map((result) => asError(result.reason))
+    if (failures.length > 0) throw new AggregateError(failures, "Reverse gateway spool cleanup failed")
   }
 
   #connect() {
@@ -249,28 +277,21 @@ export class OutboundReverseGateway extends EventEmitter {
             if (frame.type !== V3FrameType.GATEWAY_REGISTERED) throw fatalError("GATEWAY_REGISTRATION_FAILED")
             const detail = decodeV3Control(frame.payload)
             if (detail.gatewayId !== this.#options.gatewayId || detail.generation !== generation) throw fatalError("GATEWAY_REGISTRATION_FAILED")
+            if (typeof detail.resumed !== "boolean") throw fatalError("GATEWAY_REGISTRATION_FAILED")
             registered = true
-            clearTimeout(timer)
-            this.#physicalConnects += 1
-            this.#reconnectDelay = this.#options.reconnectDelayMs ?? 250
-            this.#setState("ready")
-            logDiagnostic(this.#options.logger, "transport.connected", {
-              generation,
-              attempt: this.#physicalAttempts,
-              gatewayId: this.#options.gatewayId,
-              physicalConnects: this.#physicalConnects,
-              reconnect: this.#physicalConnects > 1
-            })
-            if (this.#physicalConnects > 1) {
-              logDiagnostic(this.#options.logger, "transport.reconnect_succeeded", {
-                generation,
-                attempt: this.#reconnectAttempts,
-                gatewayId: this.#options.gatewayId,
-                physicalConnects: this.#physicalConnects
-              })
+            if (detail.resumed) {
+              // A replacement gateway intentionally owns no inherited target
+              // sockets. Its empty manifest makes the hub reset every omitted
+              // retained bridge instead of ever replaying an ambiguous OPEN.
+              clearTimeout(timer)
+              this.#resuming = true
+              await this.#sendResumeManifest(generation)
+            } else if (this.#streams.size > 0) {
+              for (const stream of [...this.#streams.values()]) this.#destroyStream(stream, "RESUME_SESSION_LOST")
+              this.#completePhysical(generation, timer)
+            } else {
+              this.#completePhysical(generation, timer)
             }
-            this.#resolveStart?.()
-            this.#settleStart()
             continue
           }
           await this.#handleFrame(frame)
@@ -290,6 +311,9 @@ export class OutboundReverseGateway extends EventEmitter {
         if (this.#physical !== physical) return
         this.#physical = undefined
         this.#codec = undefined
+        this.#resuming = false
+        this.#resumeResults = []
+        this.#resumeNextPage = 0
         if (!this.#closing && !this.#fatal) this.#setState("reconnecting")
         if (!this.#closing) {
           logDiagnostic(this.#options.logger, "transport.disconnected", {
@@ -299,8 +323,11 @@ export class OutboundReverseGateway extends EventEmitter {
             ...errorDiagnostics(attemptError, "TRANSPORT_CLOSED")
           })
         }
-        for (const stream of [...this.#streams.values()]) this.#destroyStream(stream, "GATEWAY_DISCONNECTED")
-        this.#lastStreamId = 0
+        for (const stream of this.#streams.values()) {
+          stream.sendWindow = stream.sendWindowLimit
+          for (const replay of stream.replay) replay.sent = false
+          stream.closeSent = false
+        }
         if (this.#closing) return
         if (this.#fatal) {
           this.#setState("closed")
@@ -313,6 +340,8 @@ export class OutboundReverseGateway extends EventEmitter {
           })
           this.#rejectStart?.(failure)
           this.#settleStart()
+          const failureCode = /** @type {{code?: unknown}} */ (failure).code
+          for (const stream of [...this.#streams.values()]) this.#destroyStream(stream, typeof failureCode === "string" ? failureCode : "GATEWAY_REGISTRATION_FAILED")
           return
         }
         this.#scheduleReconnect(attemptError)
@@ -324,6 +353,20 @@ export class OutboundReverseGateway extends EventEmitter {
   async #handleFrame(frame) {
     if (frame.type === V3FrameType.PING) { await this.#sendControl(V3FrameType.PONG, 0, frame.sequence, {}); return }
     if (frame.type === V3FrameType.PONG) return
+    if (frame.type === V3FrameType.RESUMED) {
+      if (!this.#resuming) throw new Error("Unexpected reverse gateway RESUMED response")
+      const manifest = decodeV3Control(frame.payload)
+      if (manifest.page !== this.#resumeNextPage || !Array.isArray(manifest.streams)) throw new Error("Invalid reverse gateway RESUMED response")
+      this.#resumeNextPage += 1
+      this.#resumeResults.push(...manifest.streams.filter((item) => typeof item === "object" && item !== null))
+      if (manifest.final !== true) return
+      for (const item of this.#resumeResults) await this.#applyResumeResult(item)
+      this.#resumeResults = []
+      this.#resumeNextPage = 0
+      this.#resuming = false
+      this.#completePhysical(this.#generation)
+      return
+    }
     if (frame.type === V3FrameType.OPEN) { await this.#openStream(frame); return }
     const stream = this.#streams.get(frame.streamId)
     if (stream === undefined) {
@@ -343,6 +386,12 @@ export class OutboundReverseGateway extends EventEmitter {
       }
       stream.receiveSequence = frame.sequence
       stream.receiveAck = frame.sequence
+      try {
+        await this.#queueSpoolOperation(stream, () => this.#spool.writeRecoveryState(stream.identifier, this.#recoveryState(stream)))
+      } catch (error) {
+        this.#resetStream(stream, spoolErrorCode(error))
+        return
+      }
       await this.#sendControl(V3FrameType.ACK, stream.id, frame.sequence, {})
       return
     }
@@ -359,6 +408,85 @@ export class OutboundReverseGateway extends EventEmitter {
     throw new Error("Unsupported reverse gateway stream frame")
   }
 
+  /** Sends the target-owning half of the paired reverse-stream resume manifest. */
+  async #sendResumeManifest(/** @type {number} */ generation) {
+    const activeStreams = [...this.#streams.values()]
+    await Promise.all(activeStreams.map((stream) => this.#queueSpoolOperation(stream, () => this.#spool.writeRecoveryState(stream.identifier, this.#recoveryState(stream)))))
+    const streams = activeStreams.filter((stream) => this.#streams.get(stream.id) === stream).map((stream) => ({
+      streamId: stream.id,
+      identifier: stream.identifier,
+      clientToGatewaySequence: stream.receiveSequence,
+      clientToGatewayAck: stream.receiveAck,
+      gatewayToClientSequence: stream.sendSequence,
+      gatewayToClientAck: stream.sendAck,
+      localEnded: stream.targetEnded,
+      remoteEnded: stream.remoteEnded
+    }))
+    this.#resumeResults = []
+    this.#resumeNextPage = 0
+    const pageCount = Math.max(1, Math.ceil(streams.length / MANIFEST_PAGE_STREAMS))
+    for (let page = 0; page < pageCount; page += 1) {
+      await this.#sendControl(V3FrameType.RESUME, 0, generation, {
+        page,
+        final: page === pageCount - 1,
+        streams: streams.slice(page * MANIFEST_PAGE_STREAMS, (page + 1) * MANIFEST_PAGE_STREAMS)
+      })
+    }
+  }
+
+  /** @param {unknown} value - Hub resume result. */
+  async #applyResumeResult(value) {
+    if (typeof value !== "object" || value === null) throw new Error("Invalid reverse gateway resume result")
+    const result = /** @type {{streamId?: unknown, status?: unknown, clientToGatewaySequence?: unknown, clientToGatewayAck?: unknown, gatewayToClientSequence?: unknown, gatewayToClientAck?: unknown, clientEnded?: unknown, gatewayEnded?: unknown, opened?: unknown, code?: unknown}} */ (value)
+    if (!Number.isInteger(result.streamId)) throw new Error("Invalid reverse gateway resume stream ID")
+    const stream = this.#streams.get(/** @type {number} */ (result.streamId))
+    if (stream === undefined) return
+    if (result.status !== "resumed") {
+      this.#destroyStream(stream, typeof result.code === "string" ? result.code : "RESUME_REJECTED")
+      return
+    }
+    const clientToGatewaySequence = /** @type {number} */ (result.clientToGatewaySequence)
+    const clientToGatewayAck = /** @type {number} */ (result.clientToGatewayAck)
+    const gatewayToClientSequence = /** @type {number} */ (result.gatewayToClientSequence)
+    const gatewayToClientAck = /** @type {number} */ (result.gatewayToClientAck)
+    const validGatewayToClientBoundary = gatewayToClientSequence === stream.sendAck || stream.replay.some((chunk) => chunk.sequence === gatewayToClientSequence)
+    if (!Number.isSafeInteger(clientToGatewaySequence) || clientToGatewaySequence !== stream.receiveSequence || !Number.isSafeInteger(clientToGatewayAck) || clientToGatewayAck < stream.receiveAck || clientToGatewayAck > clientToGatewaySequence || !Number.isSafeInteger(gatewayToClientSequence) || gatewayToClientSequence < stream.sendAck || gatewayToClientSequence > stream.sendSequence || !Number.isSafeInteger(gatewayToClientAck) || gatewayToClientAck < stream.sendAck || gatewayToClientAck > gatewayToClientSequence || !validGatewayToClientBoundary || typeof result.clientEnded !== "boolean" || typeof result.gatewayEnded !== "boolean" || typeof result.opened !== "boolean") throw new Error("Invalid reverse gateway resume positions")
+    stream.receiveAck = clientToGatewayAck
+    this.#acknowledge(stream, gatewayToClientAck)
+    stream.sendWindow = Math.max(0, stream.sendWindowLimit - (gatewayToClientSequence - gatewayToClientAck))
+    for (const chunk of stream.replay) chunk.sent = chunk.sequence <= gatewayToClientSequence
+    stream.remoteEnded = /** @type {boolean} */ (result.clientEnded)
+    stream.endSent = /** @type {boolean} */ (result.gatewayEnded)
+    stream.closeSent = false
+    if (stream.targetOpened && result.opened !== true) await this.#sendControl(V3FrameType.OPENED, stream.id, 0, {identifier: stream.identifier, initialWindow: this.#initialWindow()})
+  }
+
+  /** Marks one authenticated/registered physical generation ready. */
+  #completePhysical(/** @type {number} */ generation, /** @type {ReturnType<typeof setTimeout> | undefined} */ timer = undefined) {
+    if (timer !== undefined) clearTimeout(timer)
+    this.#physicalConnects += 1
+    this.#reconnectDelay = this.#options.reconnectDelayMs ?? 250
+    this.#setState("ready")
+    logDiagnostic(this.#options.logger, "transport.connected", {
+      generation,
+      attempt: this.#physicalAttempts,
+      gatewayId: this.#options.gatewayId,
+      physicalConnects: this.#physicalConnects,
+      reconnect: this.#physicalConnects > 1
+    })
+    if (this.#physicalConnects > 1) {
+      logDiagnostic(this.#options.logger, "transport.reconnect_succeeded", {
+        generation,
+        attempt: this.#reconnectAttempts,
+        gatewayId: this.#options.gatewayId,
+        physicalConnects: this.#physicalConnects
+      })
+    }
+    this.#resolveStart?.()
+    this.#settleStart()
+    for (const stream of this.#streams.values()) void this.#pumpStream(stream).catch(() => this.#resetStream(stream, "TARGET_READ_FAILED"))
+  }
+
   /** @param {import('../protocol/v3-frame.js').V3Frame} frame - Hub OPEN. */
   async #openStream(frame) {
     if (frame.sequence !== 0 || frame.streamId % 2 !== 0 || frame.streamId <= this.#lastStreamId) throw new Error("Invalid reverse gateway stream ID")
@@ -423,18 +551,28 @@ export class OutboundReverseGateway extends EventEmitter {
       targetClosed: false,
       targetResetCode: undefined,
       endSent: false,
+      closeSent: false,
       openTimer: undefined,
       targetOpened: false,
-      targetTerminalLogged: false
+      targetTerminalLogged: false,
+      openedReady: Promise.resolve(),
+      spoolReady: Promise.resolve(),
+      spoolQueue: Promise.resolve(),
+      spoolReservedBytes: 0,
+      spoolRemovalPromise: undefined,
+      spoolCleanupFailureRecorded: false
     }
     this.#streams.set(stream.id, stream)
+    stream.spoolReady = this.#spool.createNew({sessionId: stream.identifier, ownerId: "reverse-gateway-v3"})
+      .then(() => this.#spool.writeRecoveryState(stream.identifier, this.#recoveryState(stream)))
+    stream.spoolQueue = stream.spoolReady.catch(() => {})
+    void stream.spoolReady.catch((error) => this.#resetStream(stream, spoolErrorCode(error)))
     stream.openTimer = setTimeout(() => this.#resetStream(stream, "TARGET_OPEN_TIMEOUT"), this.#options.targetConnectTimeoutMs ?? DEFAULT_CONNECT_TIMEOUT_MS)
     stream.openTimer.unref?.()
     targetSocket.once("connect", () => {
       if (!this.#streams.has(stream.id)) return
       if (stream.openTimer !== undefined) clearTimeout(stream.openTimer)
       stream.openTimer = undefined
-      stream.state = "open"
       stream.targetOpened = true
       this.#targetOpens += 1
       logDiagnostic(this.#options.logger, "reverse_gateway.target_opened", {
@@ -442,7 +580,14 @@ export class OutboundReverseGateway extends EventEmitter {
         generation: this.#generation,
         gatewayId: this.#options.gatewayId
       })
-      void this.#sendControl(V3FrameType.OPENED, stream.id, 0, {identifier: stream.identifier, initialWindow: this.#initialWindow()})
+      stream.openedReady = stream.spoolReady.then(async () => {
+        if (!this.#streams.has(stream.id)) return
+        await this.#sendControl(V3FrameType.OPENED, stream.id, 0, {identifier: stream.identifier, initialWindow: this.#initialWindow()})
+        if (!this.#streams.has(stream.id)) return
+        stream.state = "open"
+        await this.#pumpStream(stream)
+      })
+      void stream.openedReady.catch((error) => this.#resetStream(stream, spoolErrorCode(error)))
     })
     targetSocket.on("data", (chunk) => {
       targetSocket.pause()
@@ -457,8 +602,8 @@ export class OutboundReverseGateway extends EventEmitter {
     })
     targetSocket.once("error", () => {
       if (!this.#streams.has(stream.id)) return
-      const code = stream.state === "opening" ? "TARGET_OPEN_FAILED" : "TARGET_CLOSED"
-      this.#targetOpenFailures += stream.state === "opening" ? 1 : 0
+      const code = stream.targetOpened ? "TARGET_CLOSED" : "TARGET_OPEN_FAILED"
+      this.#targetOpenFailures += stream.targetOpened ? 0 : 1
       this.#logTargetTerminal(stream, code)
       stream.targetQueue = stream.targetQueue.then(async () => {
         if (!this.#streams.has(stream.id)) return
@@ -478,7 +623,8 @@ export class OutboundReverseGateway extends EventEmitter {
 
   /** @param {GatewayStream} stream - Stream. @param {Buffer} chunk - Target output. */
   async #retainTargetData(stream, chunk) {
-    if (!this.#streams.has(stream.id) || stream.state === "opening") return
+    await stream.openedReady
+    if (!this.#streams.has(stream.id)) return
     for (let offset = 0; offset < chunk.length; offset += this.#maxFrameBytes()) {
       const data = Buffer.from(chunk.subarray(offset, Math.min(chunk.length, offset + this.#maxFrameBytes())))
       if (stream.bufferedBytes + data.length > (this.#options.maxStreamBufferBytes ?? DEFAULT_STREAM_BUFFER_BYTES) || this.#bufferedBytes + data.length > (this.#options.maxTotalBufferBytes ?? DEFAULT_TOTAL_BUFFER_BYTES)) {
@@ -489,13 +635,19 @@ export class OutboundReverseGateway extends EventEmitter {
       stream.replay.push({sequence: stream.sendSequence, data, sent: false})
       stream.bufferedBytes += data.length
       this.#bufferedBytes += data.length
+      stream.spoolReservedBytes += data.length
+      this.#spoolBytes += data.length
+      await this.#queueSpoolOperation(stream, async () => {
+        await this.#spool.appendChunk({sessionId: stream.identifier, ownerId: "reverse-gateway-v3", sequence: stream.sendSequence, data})
+        await this.#spool.writeRecoveryState(stream.identifier, this.#recoveryState(stream))
+      })
     }
     await this.#pumpStream(stream)
   }
 
   /** @param {GatewayStream} stream - Stream. */
   async #pumpStream(stream) {
-    if (stream.pumping || !this.#streams.has(stream.id) || this.#state !== "ready") return
+    if (stream.pumping || stream.state === "opening" || !this.#streams.has(stream.id) || this.#state !== "ready") return
     stream.pumping = true
     try {
       for (const chunk of stream.replay) {
@@ -513,18 +665,22 @@ export class OutboundReverseGateway extends EventEmitter {
       if (!stream.replay.some((chunk) => !chunk.sent)) {
         if (stream.targetResetCode !== undefined) {
           const code = stream.targetResetCode
+          if (!await this.#sendControl(V3FrameType.RESET, stream.id, 0, {code})) return
           stream.targetResetCode = undefined
-          await this.#sendControl(V3FrameType.RESET, stream.id, 0, {code})
           this.#destroyStream(stream, code)
           return
         }
         if (stream.targetEnded && !stream.endSent) {
           stream.endSent = true
-          await this.#sendControl(V3FrameType.END, stream.id, stream.sendSequence, {})
+          if (!await this.#sendControl(V3FrameType.END, stream.id, stream.sendSequence, {})) {
+            stream.endSent = false
+            return
+          }
         }
-        if (stream.targetClosed && (!stream.targetEnded || stream.endSent)) {
-          await this.#sendControl(V3FrameType.CLOSE, stream.id, stream.sendSequence, {})
-          this.#destroyStream(stream)
+        if (stream.targetClosed && !stream.closeSent && (!stream.targetEnded || stream.endSent)) {
+          if (!await this.#sendControl(V3FrameType.CLOSE, stream.id, stream.sendSequence, {})) return
+          stream.closeSent = true
+          stream.state = "closing"
         }
       }
     } finally {
@@ -544,6 +700,13 @@ export class OutboundReverseGateway extends EventEmitter {
     stream.bufferedBytes = Math.max(0, stream.bufferedBytes - released)
     this.#bufferedBytes = Math.max(0, this.#bufferedBytes - released)
     stream.sendWindow = Math.min(stream.sendWindowLimit, stream.sendWindow + delta)
+    void this.#queueSpoolOperation(stream, async () => {
+      await this.#spool.writeRecoveryState(stream.identifier, this.#recoveryState(stream))
+      await this.#spool.truncateAcked(stream.identifier, sequence)
+      if (released > stream.spoolReservedBytes) throw new Error("Reverse gateway spool budget release underflow")
+      stream.spoolReservedBytes -= released
+      this.#spoolBytes = Math.max(0, this.#spoolBytes - released)
+    }).catch((error) => this.#resetStream(stream, spoolErrorCode(error)))
     void this.#pumpStream(stream).catch(() => this.#resetStream(stream, "TARGET_READ_FAILED"))
   }
 
@@ -571,6 +734,7 @@ export class OutboundReverseGateway extends EventEmitter {
     stream.replay = []
     stream.bufferedBytes = 0
     stream.targetSocket.destroy()
+    this.#trackSpoolCleanup(stream)
   }
 
   /** @param {GatewayStream} stream - Stream. @param {string} code - Terminal classification. */
@@ -609,6 +773,61 @@ export class OutboundReverseGateway extends EventEmitter {
     if (!this.#referenced) this.#reconnectTimer.unref?.()
   }
 
+  /** @param {GatewayStream} stream - Durable target-side stream state. */
+  #recoveryState(stream) {
+    return {
+      version: 3,
+      role: "reverse-gateway",
+      gatewayId: this.#options.gatewayId,
+      connectionId: this.#options.connectionId,
+      streamId: stream.id,
+      identifier: stream.identifier,
+      target: stream.target,
+      receiveSequence: stream.receiveSequence,
+      receiveAck: stream.receiveAck,
+      sendSequence: stream.sendSequence,
+      sendAck: stream.sendAck,
+      remoteEnded: stream.remoteEnded,
+      targetEnded: stream.targetEnded
+    }
+  }
+
+  /** @param {GatewayStream} stream - Spool owner. @param {() => Promise<void>} operation - Serialized durable operation. */
+  async #queueSpoolOperation(stream, operation) {
+    const next = stream.spoolQueue.then(async () => {
+      await stream.spoolReady
+      await operation()
+    })
+    stream.spoolQueue = next.catch(() => {})
+    await next
+  }
+
+  /** @param {GatewayStream} stream - Terminal stream. */
+  async #removeSpool(stream) {
+    if (stream.spoolRemovalPromise === undefined) {
+      stream.spoolRemovalPromise = this.#queueSpoolOperation(stream, async () => {
+        await this.#spool.remove(stream.identifier)
+        const removed = stream.spoolReservedBytes
+        stream.spoolReservedBytes = 0
+        this.#spoolBytes = Math.max(0, this.#spoolBytes - removed)
+      })
+    }
+    await stream.spoolRemovalPromise
+  }
+
+  /** @param {GatewayStream} stream - Terminal stream. */
+  #trackSpoolCleanup(stream) {
+    const task = this.#removeSpool(stream).catch((error) => {
+      if (!stream.spoolCleanupFailureRecorded) {
+        stream.spoolCleanupFailureRecorded = true
+        this.#spoolCleanupFailures += 1
+      }
+      throw error
+    })
+    this.#spoolCleanupTasks.add(task)
+    void task.catch(() => {}).finally(() => this.#spoolCleanupTasks.delete(task))
+  }
+
   #settleStart() { this.#resolveStart = undefined; this.#rejectStart = undefined }
   #setState(/** @type {'configured' | 'connecting' | 'registering' | 'ready' | 'reconnecting' | 'closed'} */ state) { if (this.#state === state) return; this.#state = state; this.emit("stateChange", state) }
   #initialWindow() { return this.#options.initialWindowBytes ?? DEFAULT_WINDOW }
@@ -630,6 +849,8 @@ function validateOptions(options) {
   positiveInteger(options.maxStreams, "maxStreams", DEFAULT_MAX_STREAMS)
   positiveInteger(options.maxStreamBufferBytes, "maxStreamBufferBytes", DEFAULT_STREAM_BUFFER_BYTES)
   positiveInteger(options.maxTotalBufferBytes, "maxTotalBufferBytes", DEFAULT_TOTAL_BUFFER_BYTES)
+  if (options.spoolDirectory !== undefined && (typeof options.spoolDirectory !== "string" || options.spoolDirectory.length === 0)) throw new TypeError("spoolDirectory must be a non-empty string")
+  positiveInteger(options.maxAgeMs, "maxAgeMs", 24 * 60 * 60 * 1000)
   if (options.initialGeneration !== undefined && (!Number.isSafeInteger(options.initialGeneration) || options.initialGeneration < 0)) throw new RangeError("initialGeneration must be a non-negative safe integer")
 }
 
@@ -645,7 +866,7 @@ function fatalError(/** @type {string} */ code) {
 }
 
 function connectionError(/** @type {string} */ code) {
-  if (code === "HUB_SHUTDOWN") return Object.assign(new Error("Reverse gateway hub is temporarily unavailable"), {code, fatal: false})
+  if (code === "HUB_SHUTDOWN" || code === "CLIENT_DISCONNECTED" || code === "GATEWAY_DISCONNECTED") return Object.assign(new Error("Reverse gateway hub is temporarily unavailable"), {code, fatal: false})
   return fatalError(code)
 }
 
@@ -654,6 +875,12 @@ function validPeerWindow(value) { return Number.isSafeInteger(value) && /** @typ
 
 function closedError() { return Object.assign(new Error("Outbound reverse gateway is closed"), {code: "SOCKETDUCT_GATEWAY_CLOSED"}) }
 
+/** @param {unknown} error - Durable spool failure. */
+function spoolErrorCode(error) {
+  const code = /** @type {{code?: unknown}} */ (error).code
+  return typeof code === "string" ? code : "SPOOL_IO_FAILED"
+}
+
 function isFatalTransportError(/** @type {Error} */ error) {
   const code = /** @type {{code?: unknown}} */ (error).code
   return typeof code === "string" && (code.startsWith("ERR_TLS_") || code.startsWith("ERR_SSL_") || code.includes("CERT") || code.includes("VERIFY") || code.includes("SELF_SIGNED") || code.includes("ISSUER"))