socketduct 0.1.19 → 0.1.21

package/src/reverse/client.js

@@ -27,6 +27,7 @@ import {
   STABLE_IDENTIFIER,
   writeSocket
 } from "./common.js"
+import {attachDisconnectAttribution, createDisconnectAttribution, parseDisconnectAttribution, safeSocketCode} from "./disconnect-attribution.js"
 
 const DEFAULT_STREAM_BUFFER_BYTES = 4 * 1024 * 1024
 const DEFAULT_CONNECTION_BUFFER_BYTES = 32 * 1024 * 1024
@@ -62,6 +63,7 @@ const MANIFEST_PAGE_STREAMS = 64
  * @typedef {object} PendingClientWrite
  * @property {Buffer} data
  * @property {number} offset
+ * @property {boolean} durable
  * @property {(error?: Error | null) => void} callback
  * @typedef {object} ReverseClientStream
  * @property {number} id
@@ -79,6 +81,9 @@ const MANIFEST_PAGE_STREAMS = 64
  * @property {number} sendWindowLimit
  * @property {number} receiveSequence
  * @property {number} receiveAck
+ * @property {number} receiveDelivered
+ * @property {number} pendingReceiveAckBytes
+ * @property {boolean} manualReceiveAck
  * @property {Array<{sequence: number, remaining: number}>} receiveFrames
  * @property {boolean} remoteEnded
  * @property {boolean} remoteEndPending
@@ -86,8 +91,9 @@ const MANIFEST_PAGE_STREAMS = 64
  * @property {ClientChunk[]} replay
  * @property {AdaptiveCompressor} compressor
  * @property {boolean} pumping
- * @property {PendingClientWrite | undefined} pendingWrite
+ * @property {PendingClientWrite[]} pendingWrites
  * @property {boolean} progressingWrite
+ * @property {Promise<void> | undefined} writeTask
  * @property {boolean} localEnded
  * @property {boolean} endSent
  * @property {((error?: Error | null) => void) | undefined} endCallback
@@ -98,6 +104,7 @@ const MANIFEST_PAGE_STREAMS = 64
  * @property {boolean} spoolCleanupFailureRecorded
  * @property {boolean} ownsSpool
  * @property {boolean} recoveredObject
+ * @property {Promise<number>} manualAckQueue
  * @typedef {object} ReverseClientRecoveryState
  * @property {number} version
  * @property {'reverse-client'} role
@@ -119,6 +126,7 @@ const MANIFEST_PAGE_STREAMS = 64
  * @property {boolean} localEnded
  * @property {boolean} remoteEnded
  * @property {boolean} endSent
+ * @property {boolean} [manualReceiveAck]
  */
 
 /** Server3/TensorBuzz-facing virtual socket connection to an exact gateway or server-resolved route class. */
@@ -132,6 +140,7 @@ export class ReverseGatewayConnection extends EventEmitter {
   /** @type {number} */ #generation
   /** @type {string | undefined} */ #codec
   /** @type {boolean} */ #closing = false
+  /** @type {boolean} */ #detached = false
   /** @type {boolean} */ #preserveSpools = false
   /** @type {boolean} */ #fatal = false
   /** @type {Error | undefined} */ #fatalError
@@ -159,6 +168,7 @@ export class ReverseGatewayConnection extends EventEmitter {
   /** @type {number} */ #spoolBytes = 0
   /** @type {number} */ #spoolCleanupFailures = 0
   /** @type {Set<Promise<void>>} */ #spoolCleanupTasks = new Set()
+  /** @type {import('./disconnect-attribution.js').DisconnectAttribution | undefined} */ #disconnectAttribution
 
   /** @param {ReverseGatewayConnectionOptions} options - Hub route and safety policy. */
   constructor(options) {
@@ -204,9 +214,31 @@ export class ReverseGatewayConnection extends EventEmitter {
     return this.#createConnection(options, callback, true)
   }
 
+  /**
+   * Reads and validates one durable descriptor without allocating a socket.
+   * @param {string} identifier - Stable stream identifier.
+   * @returns {{identifier: string, target: {host: string, port: number}, sessionName: string, manualReceiveAck: boolean} | undefined} Recovery descriptor.
+   */
+  _inspectRecoveredConnection(identifier) {
+    if (typeof identifier !== "string" || !STABLE_IDENTIFIER.test(identifier)) throw new TypeError("Reverse virtual stream identifier must be 8-128 URL-safe characters")
+    const value = this.#spool.findRecoveryStateSync(identifier, "reverse-client-v3")
+    if (value === undefined) return undefined
+    const target = parseTarget(/** @type {{host?: unknown, port?: unknown}} */ (value.target))
+    if (target === undefined) throw new Error("Invalid reverse client recovery target")
+    const state = validateRecoveryState(value, {
+      connectionId: this.#connectionId,
+      identifier,
+      target,
+      selection: this.#routeIdentityFields(),
+      initialWindow: this.#initialWindow()
+    })
+    return {identifier, target, sessionName: state.name, manualReceiveAck: state.manualReceiveAck === true}
+  }
+
   /** Creates a fresh or synchronously recovered reverse stream. */
   #createConnection(/** @type {import('../client/multiplexed-connection.js').VirtualConnectionOptions} */ options, /** @type {((error: Error | null, socket: VirtualSocket) => void) | undefined} */ callback, /** @type {boolean} */ recovering) {
     if (this.#closing) throw closedError()
+    if (options?.manualReceiveAck !== undefined && typeof options.manualReceiveAck !== "boolean") throw new TypeError("manualReceiveAck must be a boolean")
     if (!recovering && this.#state !== "ready") throw Object.assign(new Error("Reverse gateway member is not ready"), {code: "SOCKETDUCT_TRANSPORT_SET_NOT_READY"})
     if (this.#streams.size >= (this.#options.maxStreams ?? DEFAULT_MAX_STREAMS)) throw new Error("Maximum reverse virtual stream count exceeded")
     const target = parseTarget(options?.target ?? {host: options?.host, port: options?.port})
@@ -220,7 +252,8 @@ export class ReverseGatewayConnection extends EventEmitter {
         target,
         selection: this.#routeIdentityFields(),
         initialWindow: this.#initialWindow(),
-        ...(options.sessionName === undefined ? {} : {sessionName: options.sessionName})
+        ...(options.sessionName === undefined ? {} : {sessionName: options.sessionName}),
+        ...(options.manualReceiveAck === undefined ? {} : {manualReceiveAck: options.manualReceiveAck})
       })
       : undefined
     const chunks = recovery === undefined ? [] : this.#spool.loadChunksSync(identifier, "reverse-client-v3", recovery.sendAck, recovery.sendSequence)
@@ -231,11 +264,13 @@ export class ReverseGatewayConnection extends EventEmitter {
     if (this.#streams.has(id)) throw new Error(`Reverse v3 stream ID ${id} is already active`)
     const name = options.sessionName ?? recovery?.name ?? `socketduct-reverse-${id}`
     if (typeof name !== "string" || name.length === 0 || name.length > 256) throw new TypeError("Reverse virtual stream name must be 1-256 characters")
+    const manualReceiveAck = recovery?.manualReceiveAck ?? options.manualReceiveAck ?? false
     const socket = new VirtualSocket(/** @type {import('../client/multiplexed-connection.js').MultiplexedSocketductConnection} */ (/** @type {unknown} */ (this)), id, {
       ...target,
       ...(options.allowHalfOpen === undefined ? {} : {allowHalfOpen: options.allowHalfOpen}),
       ...(options.readableHighWaterMark === undefined ? {} : {readableHighWaterMark: options.readableHighWaterMark}),
-      ...(options.writableHighWaterMark === undefined ? {} : {writableHighWaterMark: options.writableHighWaterMark})
+      ...(options.writableHighWaterMark === undefined ? {} : {writableHighWaterMark: options.writableHighWaterMark}),
+      manualReceiveAck
     })
     /** @type {ReverseClientStream} */
     const stream = {
@@ -252,6 +287,9 @@ export class ReverseGatewayConnection extends EventEmitter {
       // delivered. Resume inbound delivery from the persisted ACK boundary.
       receiveSequence: recovery?.receiveAck ?? 0,
       receiveAck: recovery?.receiveAck ?? 0,
+      receiveDelivered: recovery?.receiveAck ?? 0,
+      pendingReceiveAckBytes: 0,
+      manualReceiveAck,
       receiveFrames: [],
       remoteEnded: recovery?.remoteEnded ?? false,
       remoteEndPending: recovery?.remoteEnded ?? false,
@@ -259,8 +297,9 @@ export class ReverseGatewayConnection extends EventEmitter {
       replay: chunks.map((chunk) => ({...chunk, sent: false})),
       compressor: new AdaptiveCompressor(),
       pumping: false,
-      pendingWrite: undefined,
+      pendingWrites: [],
       progressingWrite: false,
+      writeTask: undefined,
       localEnded: recovery?.localEnded ?? false,
       endSent: recovery?.endSent ?? false,
       endCallback: undefined,
@@ -270,7 +309,8 @@ export class ReverseGatewayConnection extends EventEmitter {
       spoolRemovalPromise: undefined,
       spoolCleanupFailureRecorded: false,
       ownsSpool: recovery !== undefined,
-      recoveredObject: recovery !== undefined
+      recoveredObject: recovery !== undefined,
+      manualAckQueue: Promise.resolve(recovery?.receiveAck ?? 0)
     }
     if (recovery !== undefined) {
       this.#options.aggregateBudget?.reserveData(bufferedBytes)
@@ -367,6 +407,7 @@ export class ReverseGatewayConnection extends EventEmitter {
     if (this.#options.connectionId === undefined) throw new Error("detach() requires an explicit stable connectionId")
     if (this.#options.spoolDirectory === undefined) throw new Error("detach() requires an explicit persistent spoolDirectory")
     this.#preserveSpools = true
+    this.#detached = true
     this.#closing = true
     if (this.#reconnectTimer !== undefined) clearTimeout(this.#reconnectTimer)
     this.#reconnectTimer = undefined
@@ -377,7 +418,12 @@ export class ReverseGatewayConnection extends EventEmitter {
     const frameQueue = this.#physicalFrameQueue
     try {
       await frameQueue
-      await Promise.all([...this.#streams.values()].map((stream) => this.#persistStreamState(stream)))
+      const streams = [...this.#streams.values()]
+      await Promise.all(streams.map((stream) => stream.writeTask))
+      await Promise.all(streams.map((stream) => stream.manualAckQueue))
+      const detached = new SocketductSessionLostError("Reverse connection detached before queued write admission", "SOCKETDUCT_CONNECTION_DETACHED")
+      for (const stream of streams) while (stream.pendingWrites.length > 0) this.#finishWrite(stream, detached)
+      await Promise.all(streams.filter((stream) => this.#streams.get(stream.id) === stream).map((stream) => this.#persistStreamState(stream)))
     } finally {
       physical?.destroy()
       this.#physical = undefined
@@ -389,7 +435,15 @@ export class ReverseGatewayConnection extends EventEmitter {
   _writeStream(/** @type {number} */ streamId, /** @type {Buffer} */ data, /** @type {(error?: Error | null) => void} */ callback) {
     const stream = this.#streams.get(streamId)
     if (this.#closing || stream === undefined || stream.localEnded || stream.state === "closed" || stream.state === "reset") { callback(new SocketductStreamError("Reverse stream is not writable", "ERR_STREAM_DESTROYED")); return }
-    stream.pendingWrite = {data, offset: 0, callback}
+    stream.pendingWrites.push({data, offset: 0, durable: false, callback})
+    this.#progressPendingWrites()
+  }
+
+  _writeDurableStream(/** @type {number} */ streamId, /** @type {Buffer} */ data, /** @type {(error?: Error | null) => void} */ callback) {
+    const stream = this.#streams.get(streamId)
+    if (this.#detached) { callback(new SocketductSessionLostError("Reverse connection is detached", "SOCKETDUCT_CONNECTION_DETACHED")); return }
+    if (this.#closing || stream === undefined || stream.localEnded || stream.state === "closed" || stream.state === "reset") { callback(new SocketductStreamError("Reverse stream is not writable", "ERR_STREAM_DESTROYED")); return }
+    stream.pendingWrites.push({data, offset: 0, durable: true, callback})
     this.#progressPendingWrites()
   }
 
@@ -411,7 +465,8 @@ export class ReverseGatewayConnection extends EventEmitter {
       streamId,
       generation: this.#generation,
       ...this.#routeIdentityFields(),
-      ...(error === null ? {} : errorDiagnostics(error))
+      ...(error === null ? {} : errorDiagnostics(error)),
+      ...(error === null ? {} : publicDisconnectAttribution(error))
     })
     this.#retireStream(stream, error ?? new SocketductStreamError("Reverse stream is not writable", "ERR_STREAM_DESTROYED"))
     if (!wasClosed && !this.#closing && this.#state === "ready") void this.#sendControl(V3FrameType.RESET, streamId, 0, {code: error instanceof SocketductStreamError ? error.code : "CLIENT_CANCELLED"}).catch(() => {})
@@ -421,9 +476,17 @@ export class ReverseGatewayConnection extends EventEmitter {
     void streamId
   }
 
+  _deliverReadable(/** @type {number} */ streamId, /** @type {number} */ bytes) {
+    const stream = this.#streams.get(streamId)
+    if (stream === undefined || !stream.manualReceiveAck || !Number.isSafeInteger(bytes) || bytes <= 0) return
+    stream.receiveDelivered += bytes
+    if (stream.receiveDelivered > stream.receiveSequence) throw new Error("Reverse client readable delivery accounting overflow")
+  }
+
   _consumeReadable(/** @type {number} */ streamId, /** @type {number} */ bytes) {
     const stream = this.#streams.get(streamId)
     if (stream === undefined || !Number.isSafeInteger(bytes) || bytes <= 0) return
+    if (stream.manualReceiveAck) return
     let remaining = bytes
     let acknowledgedSequence = stream.receiveAck
     while (remaining > 0 && stream.receiveFrames.length > 0) {
@@ -440,6 +503,27 @@ export class ReverseGatewayConnection extends EventEmitter {
     this.#ackReceived(stream, acknowledgedSequence)
   }
 
+  _acknowledgeReadable(/** @type {number} */ streamId, /** @type {number} */ bytes) {
+    const stream = this.#streams.get(streamId)
+    if (this.#detached) return Promise.reject(new SocketductStreamError("Reverse connection is detached", "SOCKETDUCT_CONNECTION_DETACHED"))
+    if (this.#closing) return Promise.reject(closedError())
+    if (stream === undefined) return Promise.reject(new SocketductStreamError("Reverse stream is destroyed", "ERR_STREAM_DESTROYED"))
+    if (!stream.manualReceiveAck) return Promise.reject(new SocketductStreamError("Manual receive acknowledgement is not enabled", "SOCKETDUCT_MANUAL_ACK_DISABLED"))
+    if (stream.receiveAck + stream.pendingReceiveAckBytes + bytes > stream.receiveDelivered) throw new SocketductStreamError("Receive acknowledgement exceeds bytes delivered to the application", "SOCKETDUCT_INVALID_RECEIVE_ACK")
+    stream.pendingReceiveAckBytes += bytes
+    const acknowledged = stream.manualAckQueue.then(async () => {
+      const sequence = stream.receiveAck + bytes
+      await this.#queueSpoolOperation(stream, () => this.#spool.writeRecoveryState(stream.identifier, this.#recoveryState(stream, {receiveAck: sequence})))
+      stream.receiveAck = sequence
+      stream.pendingReceiveAckBytes -= bytes
+      await this.#sendControl(V3FrameType.ACK, stream.id, sequence, {}).catch(() => {})
+      return sequence
+    })
+    stream.manualAckQueue = acknowledged
+    void acknowledged.catch((error) => stream.socket._receiveError(asError(error)))
+    return acknowledged
+  }
+
   #allocateStreamId() {
     const id = this.#nextStreamId
     if (id > 0xffff_fffd) throw new Error("Reverse v3 stream ID space exhausted")
@@ -483,9 +567,18 @@ export class ReverseGatewayConnection extends EventEmitter {
     this.#physicalFrameQueue = queue
     let authenticated = false
     let selected = false
+    /** @type {import('./disconnect-attribution.js').DisconnectReason | undefined} */
+    let attemptDisconnectReason
+    /** @type {string | undefined} */
+    let attemptSafeSocketCode
+    /** @type {import('./disconnect-attribution.js').DisconnectAttribution | undefined} */
+    let attemptAttribution
     /** @type {Error | undefined} */
     let attemptError
-    const timer = setTimeout(() => physical.destroy(Object.assign(new Error("Reverse client TLS/auth timeout"), {code: "SOCKETDUCT_AUTH_TIMEOUT"})), this.#options.connectTimeoutMs ?? DEFAULT_CONNECT_TIMEOUT_MS)
+    const timer = setTimeout(() => {
+      attemptDisconnectReason ??= "heartbeat_timeout"
+      physical.destroy(Object.assign(new Error("Reverse client TLS/auth timeout"), {code: "SOCKETDUCT_AUTH_TIMEOUT"}))
+    }, this.#options.connectTimeoutMs ?? DEFAULT_CONNECT_TIMEOUT_MS)
     timer.unref?.()
     physical.once("secureConnect", () => {
       // The attempted generation must survive a crash after AUTH reaches the
@@ -505,13 +598,16 @@ export class ReverseGatewayConnection extends EventEmitter {
           if (this.#closing || this.#physical !== physical) return
           if (frame.type === V3FrameType.AUTH_ERROR || frame.type === V3FrameType.GOAWAY) {
             const detail = decodeV3Control(frame.payload)
-            throw connectionError(typeof detail.code === "string" ? detail.code : "CONNECTION_REJECTED")
+            if (frame.type === V3FrameType.GOAWAY) attemptDisconnectReason ??= "peer_goaway"
+            attemptAttribution = validHubAttribution(detail) ?? attemptAttribution
+            throw connectionError(typeof detail.code === "string" ? detail.code : "CONNECTION_REJECTED", attemptAttribution)
           }
           if (!authenticated) {
             if (frame.type !== V3FrameType.AUTH_OK) throw fatalError("AUTH_NEGOTIATION_FAILED")
             const detail = decodeV3Control(frame.payload)
             if (detail.version !== 3 || detail.role !== "client" || detail.connectionId !== this.#connectionId || detail.generation !== generation || detail.compression !== ZSTD_CODEC || !validPeerWindow(detail.initialWindow)) throw fatalError("AUTH_NEGOTIATION_FAILED")
             authenticated = true
+            this.#disconnectAttribution = undefined
             this.#codec = ZSTD_CODEC
             this.#sendWindowLimit = Math.min(this.#initialWindow(), detail.initialWindow)
             for (const stream of this.#streams.values()) {
@@ -544,12 +640,19 @@ export class ReverseGatewayConnection extends EventEmitter {
         }
       }).catch((error) => {
         attemptError = asError(error)
+        attemptDisconnectReason ??= "protocol_violation"
         if (/** @type {{fatal?: unknown}} */ (attemptError).fatal === true) this.#fatal = true
         physical.destroy(attemptError)
       }).finally(() => { if (!physical.destroyed) physical.resume() })
       this.#physicalFrameQueue = queue
     })
-    physical.once("error", (error) => { attemptError = error; if (isFatalTransportError(error)) this.#fatal = true })
+    physical.once("end", () => { attemptDisconnectReason ??= "remote_eof" })
+    physical.once("error", (error) => {
+      attemptError = error
+      attemptDisconnectReason ??= "remote_socket_error"
+      attemptSafeSocketCode ??= safeSocketCode(error)
+      if (isFatalTransportError(error)) this.#fatal = true
+    })
     physical.once("close", () => {
       clearTimeout(timer)
       void queue.finally(() => {
@@ -561,11 +664,28 @@ export class ReverseGatewayConnection extends EventEmitter {
         this.#resumeNextPage = 0
         if (!this.#closing && !this.#fatal) this.#setState("reconnecting")
         const lost = new SocketductSessionLostError("Reverse gateway physical transport was interrupted", "GATEWAY_DISCONNECTED")
+        const attribution = attemptAttribution ?? this.#disconnectAttribution ?? createDisconnectAttribution({
+          disconnectReason: attemptDisconnectReason ?? "unknown_transport_close",
+          incidentOrigin: "reverse_client",
+          physicalRole: "client",
+          clientConnectionGeneration: generation,
+          fanOutCount: this.#streams.size,
+          resumeSuccessCount: 0,
+          terminalStreamCount: 0
+        })
+        this.#disconnectAttribution = attribution
         if (!this.#closing) {
+          logDiagnostic(this.#options.logger, "reverse_client.physical_detached", {
+            ...this.#routeIdentityFields(),
+            ...attribution,
+            ...(attemptSafeSocketCode === undefined ? {} : {safeSocketCode: attemptSafeSocketCode})
+          })
           logDiagnostic(this.#options.logger, "transport.disconnected", {
-            generation,
+            clientConnectionGeneration: generation,
             ...this.#routeIdentityFields(),
             virtualStreams: this.#streams.size,
+            ...attribution,
+            ...(attemptSafeSocketCode === undefined ? {} : {safeSocketCode: attemptSafeSocketCode}),
             ...errorDiagnostics(attemptError ?? lost, "GATEWAY_DISCONNECTED")
           })
         }
@@ -647,7 +767,7 @@ export class ReverseGatewayConnection extends EventEmitter {
       if (this.#closing || this.#physical !== physical) return
       if (data.length === 0 || data.length > this.#maxFrameBytes() || frame.sequence !== stream.receiveSequence + data.length || frame.sequence - stream.receiveAck > this.#initialWindow()) throw new Error("Invalid reverse client DATA sequence or window")
       stream.receiveSequence = frame.sequence
-      stream.receiveFrames.push({sequence: frame.sequence, remaining: data.length})
+      if (!stream.manualReceiveAck) stream.receiveFrames.push({sequence: frame.sequence, remaining: data.length})
       stream.socket._receive(data)
       return
     }
@@ -681,7 +801,7 @@ export class ReverseGatewayConnection extends EventEmitter {
       const detail = decodeV3Control(frame.payload)
       const code = typeof detail.code === "string" ? detail.code : "STREAM_RESET"
       stream.state = "reset"
-      const error = new SocketductStreamError("Reverse gateway stream was reset", code)
+      const error = attachDisconnectAttribution(new SocketductStreamError("Reverse gateway stream was reset", code), validHubAttribution(detail) ?? (code === "GATEWAY_DISCONNECTED" ? this.#disconnectAttribution : undefined))
       stream.socket._receiveError(error)
       return
     }
@@ -725,7 +845,7 @@ export class ReverseGatewayConnection extends EventEmitter {
     if (stream === undefined) return
     if (result.status !== "resumed") {
       const code = typeof result.code === "string" ? result.code : "RESUME_REJECTED"
-      stream.socket._receiveError(new SocketductSessionLostError("Reverse gateway stream could not be resumed", code))
+      stream.socket._receiveError(attachDisconnectAttribution(new SocketductSessionLostError("Reverse gateway stream could not be resumed", code), validHubAttribution(result) ?? this.#disconnectAttribution))
       return
     }
     const clientToGatewaySequence = /** @type {number} */ (result.clientToGatewaySequence)
@@ -820,10 +940,18 @@ export class ReverseGatewayConnection extends EventEmitter {
 
   /** @param {ReverseClientStream} stream - Stream with an active application write. */
   async #progressWrite(stream) {
-    const pending = stream.pendingWrite
+    const pending = stream.pendingWrites[0]
     if (pending === undefined || stream.progressingWrite || stream.state === "opening") return
     stream.progressingWrite = true
     try {
+      if (pending.durable) {
+        await this.#admitDurableWrite(stream, pending)
+        if (stream.pendingWrites[0] === pending) this.#finishWrite(stream)
+        queueMicrotask(() => {
+          void this.#pumpStream(stream).catch((error) => stream.socket._receiveError(asError(error)))
+        })
+        return
+      }
       if (pending.offset < pending.data.length && this.#streams.get(stream.id) === stream) {
         const available = this.#writeCapacity(stream)
         if (available <= 0) return
@@ -847,17 +975,65 @@ export class ReverseGatewayConnection extends EventEmitter {
         })
       }
       await this.#pumpStream(stream)
-      if (pending.offset === pending.data.length && stream.pendingWrite === pending) this.#finishWrite(stream)
+      if (pending.offset === pending.data.length && stream.pendingWrites[0] === pending) this.#finishWrite(stream)
     } catch (error) {
       const failure = asError(error)
       this.#finishWrite(stream, failure)
-      stream.socket._receiveError(failure)
+      if (!pending.durable) stream.socket._receiveError(failure)
     } finally {
       stream.progressingWrite = false
       this.#progressPendingWrites()
     }
   }
 
+  /** @param {ReverseClientStream} stream - Stream. @param {PendingClientWrite} pending - Complete durable operation. */
+  async #admitDurableWrite(stream, pending) {
+    const bytes = pending.data.length
+    if (bytes === 0) return
+    const streamCapacity = (this.#options.maxStreamBufferBytes ?? DEFAULT_STREAM_BUFFER_BYTES) - stream.bufferedBytes
+    const connectionCapacity = (this.#options.maxConnectionBufferBytes ?? DEFAULT_CONNECTION_BUFFER_BYTES) - this.#bufferedBytes
+    if (bytes > streamCapacity || bytes > connectionCapacity || !Number.isSafeInteger(stream.sendSequence + bytes)) throw new SocketductStreamError("Complete durable write exceeds available retained-byte capacity", "SOCKETDUCT_DURABLE_WRITE_LIMIT")
+    let reserved = false
+    try {
+      this.#options.aggregateBudget?.reserveData(bytes)
+      reserved = true
+      let sequence = stream.sendSequence
+      /** @type {ClientChunk[]} */
+      const chunks = []
+      for (let offset = 0; offset < bytes; offset += this.#maxFrameBytes()) {
+        const data = Buffer.from(pending.data.subarray(offset, Math.min(bytes, offset + this.#maxFrameBytes())))
+        sequence += data.length
+        chunks.push({sequence, data, sent: false})
+      }
+      await this.#queueSpoolOperation(stream, () => this.#spool.appendChunksAndRecoveryState({
+        sessionId: stream.identifier,
+        ownerId: "reverse-client-v3",
+        previousSequence: stream.sendSequence,
+        chunks,
+        recoveryState: this.#recoveryState(stream, {sendSequence: sequence})
+      }))
+      if (this.#streams.get(stream.id) !== stream) throw new SocketductStreamError("Reverse stream was destroyed during durable admission", "ERR_STREAM_DESTROYED")
+      stream.sendSequence = sequence
+      stream.replay.push(...chunks)
+      stream.bufferedBytes += bytes
+      this.#bufferedBytes += bytes
+      stream.spoolReservedBytes += bytes
+      this.#spoolBytes += bytes
+      pending.offset = bytes
+    } catch (error) {
+      if (reserved) {
+        this.#options.aggregateBudget?.releaseMemory(bytes)
+        this.#options.aggregateBudget?.releaseSpool(bytes)
+      }
+      const failure = asError(error)
+      const code = /** @type {{code?: unknown}} */ (failure).code
+      if (failure instanceof SocketductStreamError || (typeof code === "string" && code.startsWith("SOCKETDUCT_SET_"))) throw failure
+      const wrapped = new SocketductStreamError("Durable write persistence failed before admission", "SOCKETDUCT_DURABLE_WRITE_PERSISTENCE")
+      wrapped.cause = failure
+      throw wrapped
+    }
+  }
+
   /** @param {ReverseClientStream} stream - Stream. @returns {number} Retained-byte headroom for its next frame. */
   #writeCapacity(stream) {
     return Math.max(0, Math.min(
@@ -870,25 +1046,27 @@ export class ReverseGatewayConnection extends EventEmitter {
 
   /** Starts one bounded pending-write slice using rotating stream priority. */
   #progressPendingWrites() {
+    if (this.#closing) return
     const streams = [...this.#streams.values()]
     if (streams.length === 0) return
     const previousIndex = streams.findIndex((stream) => stream.id === this.#lastProgressedStreamId)
     const startIndex = previousIndex === -1 ? 0 : (previousIndex + 1) % streams.length
     for (let offset = 0; offset < streams.length; offset += 1) {
       const stream = /** @type {ReverseClientStream} */ (streams[(startIndex + offset) % streams.length])
-      const pending = stream.pendingWrite
-      if (pending === undefined || stream.progressingWrite || stream.state === "opening" || (pending.offset < pending.data.length && this.#writeCapacity(stream) <= 0)) continue
+      const pending = stream.pendingWrites[0]
+      if (pending === undefined || stream.progressingWrite || stream.state === "opening" || (!pending.durable && pending.offset < pending.data.length && this.#writeCapacity(stream) <= 0)) continue
       this.#lastProgressedStreamId = stream.id
-      void this.#progressWrite(stream)
+      const task = this.#progressWrite(stream)
+      stream.writeTask = task
+      void task.finally(() => { if (stream.writeTask === task) stream.writeTask = undefined })
       return
     }
   }
 
   /** @param {ReverseClientStream} stream - Stream. @param {Error} [error] - Write failure. */
   #finishWrite(stream, error) {
-    const pending = stream.pendingWrite
+    const pending = stream.pendingWrites.shift()
     if (pending === undefined) return
-    stream.pendingWrite = undefined
     pending.callback(error)
   }
 
@@ -906,7 +1084,7 @@ export class ReverseGatewayConnection extends EventEmitter {
     this.#options.aggregateBudget?.releaseMemory(stream.bufferedBytes)
     stream.replay = []
     stream.bufferedBytes = 0
-    this.#finishWrite(stream, pendingWriteError)
+    while (stream.pendingWrites.length > 0) this.#finishWrite(stream, pendingWriteError)
     if (!this.#preserveSpools) this.#trackSpoolCleanup(stream)
     this.#progressPendingWrites()
   }
@@ -918,7 +1096,16 @@ export class ReverseGatewayConnection extends EventEmitter {
     const delta = sequence - stream.sendAck
     stream.sendAck = sequence
     let released = 0
-    stream.replay = stream.replay.filter((chunk) => { if (chunk.sequence <= sequence) { released += chunk.data.length; return false } return true })
+    stream.replay = stream.replay.flatMap((chunk) => {
+      const start = chunk.sequence - chunk.data.length
+      if (chunk.sequence <= sequence) { released += chunk.data.length; return [] }
+      if (start < sequence) {
+        const acknowledgedBytes = sequence - start
+        released += acknowledgedBytes
+        return [{...chunk, data: Buffer.from(chunk.data.subarray(acknowledgedBytes))}]
+      }
+      return [chunk]
+    })
     stream.bufferedBytes = Math.max(0, stream.bufferedBytes - released)
     this.#bufferedBytes = Math.max(0, this.#bufferedBytes - released)
     this.#options.aggregateBudget?.releaseMemory(released)
@@ -954,7 +1141,19 @@ export class ReverseGatewayConnection extends EventEmitter {
   /** @param {number} type - Type. @param {number} streamId - Stream. @param {number} sequence - Sequence. @param {Record<string, unknown>} body - Body. */
   async #sendControl(type, streamId, sequence, body) {
     const physical = this.#physical
-    if (physical === undefined || physical.destroyed) throw new SocketductSessionLostError("Reverse gateway transport is unavailable", "GATEWAY_DISCONNECTED")
+    if (physical === undefined || physical.destroyed) {
+      this.#disconnectAttribution ??= createDisconnectAttribution({
+        disconnectReason: "send_after_transport_loss",
+        incidentOrigin: "reverse_client",
+        physicalRole: "client",
+        clientConnectionGeneration: this.#generation,
+        fanOutCount: this.#streams.size,
+        resumeSuccessCount: 0,
+        terminalStreamCount: 0
+      })
+      logDiagnostic(this.#options.logger, "reverse_client.send_after_transport_loss", {...this.#routeIdentityFields(), ...this.#disconnectAttribution})
+      throw attachDisconnectAttribution(new SocketductSessionLostError("Reverse gateway transport is unavailable", "GATEWAY_DISCONNECTED"), this.#disconnectAttribution)
+    }
     return await writeSocket(physical, encodeV3Frame({type, streamId, sequence}, encodeV3Control(body)))
   }
 
@@ -986,8 +1185,8 @@ export class ReverseGatewayConnection extends EventEmitter {
     }
   }
 
-  /** @param {ReverseClientStream} stream - Durable stream state. */
-  #recoveryState(stream) {
+  /** @param {ReverseClientStream} stream - Durable stream state. @param {{sendSequence?: number, receiveAck?: number}} [overrides] - Commit-boundary overrides. */
+  #recoveryState(stream, overrides = {}) {
     return {
       version: 3,
       role: "reverse-client",
@@ -1001,10 +1200,11 @@ export class ReverseGatewayConnection extends EventEmitter {
       state: stream.state,
       opened: stream.opened,
       openingSent: stream.openingSent,
-      sendSequence: stream.sendSequence,
+      sendSequence: overrides.sendSequence ?? stream.sendSequence,
       sendAck: stream.sendAck,
       receiveSequence: stream.receiveSequence,
-      receiveAck: stream.receiveAck,
+      receiveAck: overrides.receiveAck ?? stream.receiveAck,
+      manualReceiveAck: stream.manualReceiveAck,
       localEnded: stream.localEnded,
       remoteEnded: stream.remoteEnded,
       endSent: stream.endSent
@@ -1094,7 +1294,7 @@ function validateOptions(options) {
 
 /**
  * @param {Record<string, unknown>} value - Parsed durable state.
- * @param {{connectionId: string, identifier: string, target: {host: string, port: number}, selection: {gatewayId: string} | {routeClass: string}, initialWindow: number, sessionName?: string}} expected - Caller-owned recovery identity.
+ * @param {{connectionId: string, identifier: string, target: {host: string, port: number}, selection: {gatewayId: string} | {routeClass: string}, initialWindow: number, sessionName?: string, manualReceiveAck?: boolean}} expected - Caller-owned recovery identity.
  * @returns {ReverseClientRecoveryState} Validated state.
  */
 function validateRecoveryState(value, expected) {
@@ -1127,6 +1327,7 @@ function validateRecoveryState(value, expected) {
     || typeof state.localEnded !== "boolean"
     || typeof state.remoteEnded !== "boolean"
     || typeof state.endSent !== "boolean"
+    || (state.manualReceiveAck !== undefined && typeof state.manualReceiveAck !== "boolean")
     || (state.endSent && !state.localEnded)
     || (state.endSent && state.opened !== true)
     || (state.remoteEnded && state.opened !== true)
@@ -1136,6 +1337,7 @@ function validateRecoveryState(value, expected) {
     || /** @type {number} */ (state.receiveAck) > /** @type {number} */ (state.receiveSequence)
     || /** @type {number} */ (state.receiveSequence) - /** @type {number} */ (state.receiveAck) > expected.initialWindow
     || (expected.sessionName !== undefined && state.name !== expected.sessionName)
+    || (expected.manualReceiveAck !== undefined && (state.manualReceiveAck === true) !== expected.manualReceiveAck)
   ) throw new Error("Invalid reverse client recovery identity, selector, lifecycle, or sequence state")
   return /** @type {ReverseClientRecoveryState} */ (state)
 }
@@ -1168,9 +1370,18 @@ function installCallback(socket, callback) {
 }
 
 function fatalError(/** @type {string} */ code) { return Object.assign(new Error("Reverse gateway client was rejected by the hub"), {code, fatal: true}) }
-function connectionError(/** @type {string} */ code) {
-  if (code === "HUB_SHUTDOWN" || code === "GATEWAY_UNAVAILABLE" || code === "GATEWAY_DISCONNECTED" || code === "GATEWAY_REPLACED") return Object.assign(new Error("Selected reverse gateway is temporarily unavailable"), {code, fatal: false})
-  return fatalError(code)
+function connectionError(/** @type {string} */ code, /** @type {unknown} */ attribution = undefined) {
+  if (code === "HUB_SHUTDOWN" || code === "GATEWAY_UNAVAILABLE" || code === "GATEWAY_DISCONNECTED" || code === "GATEWAY_REPLACED") return attachDisconnectAttribution(Object.assign(new Error("Selected reverse gateway is temporarily unavailable"), {code, fatal: false}), attribution)
+  return attachDisconnectAttribution(fatalError(code), attribution)
+}
+/** Accepts attribution only from the authenticated hub transport. */
+function validHubAttribution(/** @type {unknown} */ value) {
+  const attribution = parseDisconnectAttribution(value)
+  return attribution?.incidentOrigin === "reverse_hub" ? attribution : undefined
+}
+/** Extracts only the closed public attribution contract from an error. */
+function publicDisconnectAttribution(/** @type {unknown} */ error) {
+  return parseDisconnectAttribution(error) ?? {}
 }
 /** @param {unknown} value - Advertised peer receive window. @returns {value is number} Whether valid. */
 function validPeerWindow(value) { return Number.isSafeInteger(value) && /** @type {number} */ (value) > 0 }