socketduct 0.1.17 → 0.1.18

package/test/integration/reverse-gateway.test.js

@@ -1,6 +1,6 @@
 /* eslint-disable jsdoc/escape-inline-tags, jsdoc/require-param, jsdoc/require-returns */
 import assert from "node:assert/strict"
-import {mkdtemp, rm} from "node:fs/promises"
+import {mkdtemp, readdir, rm} from "node:fs/promises"
 import {connect as connectTcp, createServer} from "node:net"
 import {tmpdir} from "node:os"
 import {join} from "node:path"
@@ -10,7 +10,7 @@ import {connect as connectTls} from "node:tls"
 
 import {StaticTokenStore} from "../../src/auth/static-token-store.js"
 import {MultiplexedSocketductTransportSet} from "../../src/client/multiplexed-transport-set.js"
-import {encodeV3Control, encodeV3Frame, V3FrameType} from "../../src/protocol/v3-frame.js"
+import {encodeV3Control, encodeV3Frame, V3FrameDecoder, V3FrameType} from "../../src/protocol/v3-frame.js"
 import {OutboundReverseGateway, ReverseGatewayConnection, startReverseGatewayHub} from "../../src/reverse/index.js"
 import {generateTestCertificates} from "../helpers/tls-certificates.js"
 
@@ -151,6 +151,7 @@ class HeldWriteTarget extends Duplex {
 
 class ObservedMemoryBudget {
   /** @type {number} */ bytes = 0
+  /** @type {number} */ spoolBytes = 0
   /** @type {number} */ peakBytes = 0
 
   /** @param {number} maxBytes - Hard reservation ceiling. */
@@ -159,9 +160,10 @@ class ObservedMemoryBudget {
   }
 
   /** @param {number} bytes - Reverse-stream bytes to retain. */
-  reserveMemory(bytes) {
+  reserveData(bytes) {
     assert(this.bytes + bytes <= this.maxBytes, "reverse client exceeded the aggregate retained-byte ceiling")
     this.bytes += bytes
+    this.spoolBytes += bytes
     this.peakBytes = Math.max(this.peakBytes, this.bytes)
   }
 
@@ -171,8 +173,11 @@ class ObservedMemoryBudget {
     this.bytes -= bytes
   }
 
-  reserveData() { assert.fail("reverse client must reserve non-durable bytes as memory") }
-  releaseSpool() { assert.fail("reverse client must not reserve spool bytes") }
+  reserveMemory() { assert.fail("reverse client replay must be durably reserved") }
+  releaseSpool(/** @type {number} */ bytes) {
+    assert(bytes <= this.spoolBytes, "reverse client released more spool bytes than it reserved")
+    this.spoolBytes -= bytes
+  }
   reserveControl() {}
   releaseControl() {}
 }
@@ -346,6 +351,23 @@ class DataThenLocalCloseTarget extends Duplex {
   }
 }
 
+class InterruptibleTarget extends Duplex {
+  constructor() {
+    super()
+    setImmediate(() => this.emit("connect"))
+  }
+
+  /** @override */
+  _read() {}
+
+  /** @param {Buffer} _chunk - Ignored bytes. @param {BufferEncoding} _encoding - Ignored encoding. @param {(error?: Error | null) => void} callback - Completion. @override */
+  _write(_chunk, _encoding, callback) { callback() }
+
+  fail() { this.emit("error", new Error("target disappeared during reverse transport interruption")) }
+
+  closeLocally() { this.emit("end"); this.emit("close") }
+}
+
 test("reverse gateway registers outbound over TLS and multiplexes compressed streams", async (t) => {
   /** @type {Array<{event: string} & Record<string, unknown>>} */
   const gatewayLifecycle = []
@@ -1155,49 +1177,575 @@ test("hub and gateway both enforce target policy without opening a proxy", async
   allowed.destroy()
 })
 
-test("gateway loss terminates uncertain streams, reconnects one owner, and accepts clean new work", async (t) => {
-  const root = await mkdtemp(join(tmpdir(), "socketduct-reverse-reconnect-"))
+test("gateway transport replacement resumes one established reverse stream exactly once", async (t) => {
+  const root = await mkdtemp(join(tmpdir(), "socketduct-reverse-resume-"))
   t.after(() => rm(root, {recursive: true, force: true}))
   const certificates = generateTestCertificates(root)
+  const clientSpoolDirectory = join(root, "client-spool")
+  const gatewaySpoolDirectory = join(root, "gateway-spool")
+  /** @type {import('node:net').Socket | undefined} */
+  let targetSocket
+  let targetAccepts = 0
+  /** @type {Buffer[]} */
+  const targetReceived = []
   const targetSockets = new Set()
   const target = createServer((socket) => {
+    targetAccepts += 1
+    targetSocket = socket
     targetSockets.add(socket)
     socket.once("close", () => targetSockets.delete(socket))
-    socket.on("data", (chunk) => socket.write(chunk))
+    socket.on("data", (chunk) => targetReceived.push(Buffer.from(chunk)))
   })
   const targetPort = await listen(target)
   t.after(() => {
     for (const socket of targetSockets) socket.destroy()
     return new Promise((resolve) => target.close(() => resolve(undefined)))
   })
+
   const tokens = new StaticTokenStore([{username: "home", token: "gateway-token"}, {username: "app", token: "client-token"}])
-  const hub = await startReverseGatewayHub({tokens, tls: {certFile: certificates.serverCertFile, keyFile: certificates.serverKeyFile}, authorizeGateway: () => true, authorizeClientGateway: () => true, authorizeOpen: () => true})
+  const hub = await startReverseGatewayHub({
+    tokens,
+    tls: {certFile: certificates.serverCertFile, keyFile: certificates.serverKeyFile},
+    knownGatewayIds: ["minisforum-starlink"],
+    knownRouteClasses: ["starlink"],
+    authorizeGateway: () => true,
+    authorizeClientGateway: () => false,
+    authorizeClientRouteClass: () => true,
+    resolveOpenGateways: () => ["minisforum-starlink"],
+    authorizeOpen: () => true,
+    resumeWindowMs: 2_000
+  })
+  t.after(() => hub.close())
+  const tls = {caFile: certificates.caCertFile, servername: "localhost"}
+  let holdNextGatewayData = false
+  /** @type {() => void} */
+  let markGatewayDataHeld = () => {}
+  const gatewayDataHeld = new Promise((resolve) => { markGatewayDataHeld = () => resolve(undefined) })
+  const gateway = new OutboundReverseGateway({
+    gatewayId: "minisforum-starlink",
+    connectionId: "resume_gateway_01",
+    hub: {host: "127.0.0.1", port: hub.port, token: "gateway-token"},
+    tls,
+    spoolDirectory: gatewaySpoolDirectory,
+    authorizeTarget: () => true,
+    reconnectDelayMs: 50,
+    maxReconnectDelayMs: 50,
+    createPhysicalSocket: (options) => {
+      const physical = connectTls({host: options.host, port: options.port, ...options.tls})
+      const originalWrite = physical.write.bind(physical)
+      physical.write = /** @type {typeof physical.write} */ ((chunk, ...args) => {
+        if (holdNextGatewayData && Buffer.isBuffer(chunk) && chunk[5] === V3FrameType.DATA) {
+          holdNextGatewayData = false
+          markGatewayDataHeld()
+          return true
+        }
+        return Reflect.apply(originalWrite, physical, [chunk, ...args])
+      })
+      return physical
+    }
+  })
+  t.after(() => gateway.close())
+  await gateway.start()
+  const client = new ReverseGatewayConnection({
+    routeClass: "starlink",
+    connectionId: "resume_client_01",
+    hub: {host: "127.0.0.1", port: hub.port, token: "client-token"},
+    tls,
+    spoolDirectory: clientSpoolDirectory,
+    reconnectDelayMs: 10,
+    maxReconnectDelayMs: 20
+  })
+  t.after(() => client.close())
+  await client.connect()
+
+  const socket = client.createConnection({host: "127.0.0.1", port: targetPort, sessionName: "docker-exec"})
+  /** @type {Buffer[]} */
+  const clientReceived = []
+  /** @type {Error[]} */
+  const clientErrors = []
+  let clientEnds = 0
+  let clientCloses = 0
+  socket.on("data", (chunk) => clientReceived.push(Buffer.from(chunk)))
+  socket.on("error", (error) => clientErrors.push(error))
+  socket.on("end", () => { clientEnds += 1 })
+  socket.on("close", () => { clientCloses += 1 })
+  await onceEvent(socket, "connect")
+  assert(targetSocket !== undefined)
+
+  socket.write("client-before|")
+  await waitUntil(() => Buffer.concat(targetReceived).toString() === "client-before|", "initial client bytes")
+  targetSocket.write("server-before|")
+  await waitUntil(() => Buffer.concat(clientReceived).toString() === "server-before|", "initial target bytes")
+
+  holdNextGatewayData = true
+  targetSocket.write("server-during|")
+  await withTimeout(gatewayDataHeld, "unacknowledged gateway DATA")
+  await hub.closeActiveGateway("minisforum-starlink")
+  await waitUntil(() => gateway.status().physicalState === "reconnecting", "gateway interruption")
+  await new Promise((resolve) => setTimeout(resolve, 20))
+  assert.equal(clientErrors.length, 0, "transport loss must not error the app-facing socket")
+  assert.equal(clientEnds, 0, "transport loss must not end the app-facing socket")
+  assert.equal(clientCloses, 0, "transport loss must not close the app-facing socket")
+
+  const duringWrite = new Promise((resolve, reject) => socket.write("client-during|", (error) => error ? reject(error) : resolve(undefined)))
+  await waitUntil(() => gateway.status().physicalConnects >= 2 && gateway.status().physicalState === "ready", "gateway transport replacement")
+  await withTimeout(duringWrite, "client write retained during disconnect")
+  await waitUntil(() => Buffer.concat(targetReceived).toString() === "client-before|client-during|", "client bytes replayed after resume")
+  await waitUntil(() => Buffer.concat(clientReceived).toString() === "server-before|server-during|", "unacknowledged target bytes replayed after resume")
+
+  socket.write("client-after|")
+  targetSocket.write("server-after|")
+  await waitUntil(() => Buffer.concat(targetReceived).toString() === "client-before|client-during|client-after|", "post-resume client bytes")
+  await waitUntil(() => Buffer.concat(clientReceived).toString() === "server-before|server-during|server-after|", "post-resume target bytes")
+  assert.equal(targetAccepts, 1, "resume must retain the original logical target stream")
+  assert.equal(clientErrors.length, 0)
+  assert.equal(clientEnds, 0)
+  assert.equal(clientCloses, 0)
+
+  const closed = onceEvent(socket, "close")
+  socket.end()
+  await withTimeout(closed, "resumed reverse stream cleanup")
+  await waitUntil(() => hub.status().virtualStreams === 0 && gateway.status().virtualStreams === 0 && client.status().virtualStreams === 0 && gateway.status().spoolBytes === 0 && client.status().spoolBytes === 0 && targetSockets.size === 0, "resumed stream ownership cleanup")
+  assert.equal(clientErrors.length, 0)
+  assert.equal(clientEnds, 1)
+  assert.equal(clientCloses, 1)
+  assert.deepEqual(await readdir(clientSpoolDirectory), [])
+  assert.deepEqual(await readdir(gatewaySpoolDirectory), [])
+})
+
+test("gateway retains final response replay until resumed CLOSE is acknowledged", async (t) => {
+  const root = await mkdtemp(join(tmpdir(), "socketduct-reverse-resume-close-"))
+  t.after(() => rm(root, {recursive: true, force: true}))
+  const certificates = generateTestCertificates(root)
+  const payload = Buffer.from("final-response-before-transport-loss")
+  const tokens = new StaticTokenStore([{username: "home", token: "gateway-token"}, {username: "app", token: "client-token"}])
+  const hub = await startReverseGatewayHub({tokens, tls: {certFile: certificates.serverCertFile, keyFile: certificates.serverKeyFile}, authorizeGateway: () => true, authorizeClientGateway: () => true, authorizeOpen: () => true, resumeWindowMs: 2_000})
+  t.after(() => hub.close())
+  const tls = {caFile: certificates.caCertFile, servername: "localhost"}
+  /** @type {DataThenLocalCloseTarget | undefined} */
+  let target
+  /** @type {() => void} */
+  let markCloseHeld = () => {}
+  const closeHeld = new Promise((resolve) => { markCloseHeld = () => resolve(undefined) })
+  let holdGatewayClose = true
+  const gateway = new OutboundReverseGateway({
+    gatewayId: "o2",
+    connectionId: "resume_close_gateway",
+    hub: {host: "127.0.0.1", port: hub.port, token: "gateway-token"},
+    tls,
+    spoolDirectory: join(root, "gateway-spool"),
+    authorizeTarget: () => true,
+    reconnectDelayMs: 20,
+    maxReconnectDelayMs: 20,
+    createTargetConnection: () => {
+      target = new DataThenLocalCloseTarget(payload)
+      return /** @type {import('node:net').Socket} */ (/** @type {unknown} */ (target))
+    },
+    createPhysicalSocket: (options) => {
+      const physical = connectTls({host: options.host, port: options.port, ...options.tls})
+      const originalWrite = physical.write.bind(physical)
+      physical.write = /** @type {typeof physical.write} */ ((chunk, ...args) => {
+        if (holdGatewayClose && Buffer.isBuffer(chunk) && chunk[5] === V3FrameType.CLOSE) {
+          holdGatewayClose = false
+          markCloseHeld()
+          return true
+        }
+        return Reflect.apply(originalWrite, physical, [chunk, ...args])
+      })
+      return physical
+    }
+  })
+  t.after(() => gateway.close())
+  await gateway.start()
+  /** @type {() => void} */
+  let markAckHeld = () => {}
+  const ackHeld = new Promise((resolve) => { markAckHeld = () => resolve(undefined) })
+  let holdClientAck = true
+  const client = new ReverseGatewayConnection({
+    gatewayId: "o2",
+    connectionId: "resume_close_client",
+    hub: {host: "127.0.0.1", port: hub.port, token: "client-token"},
+    tls,
+    spoolDirectory: join(root, "client-spool"),
+    reconnectDelayMs: 10,
+    maxReconnectDelayMs: 20,
+    createPhysicalSocket: (options) => {
+      const physical = connectTls({host: options.host, port: options.port, ...options.tls})
+      const originalWrite = physical.write.bind(physical)
+      physical.write = /** @type {typeof physical.write} */ ((chunk, ...args) => {
+        if (holdClientAck && Buffer.isBuffer(chunk) && chunk[5] === V3FrameType.ACK) {
+          holdClientAck = false
+          markAckHeld()
+          return true
+        }
+        return Reflect.apply(originalWrite, physical, [chunk, ...args])
+      })
+      return physical
+    }
+  })
+  t.after(() => client.close())
+  await client.connect()
+
+  const socket = client.createConnection({host: "closing-target", port: 2375, allowHalfOpen: true})
+  const response = readToEnd(socket)
+  /** @type {Error[]} */
+  const errors = []
+  socket.on("error", (error) => errors.push(error))
+  const closed = onceEvent(socket, "close")
+  await onceEvent(socket, "connect")
+  await withTimeout(ackHeld, "final response ACK held")
+  assert(target !== undefined)
+  target.closeLocally()
+  await withTimeout(closeHeld, "gateway CLOSE held before transport loss")
+  assert.equal(gateway.status().virtualStreams, 1, "gateway must retain the closing stream until echoed CLOSE")
+  assert.equal(gateway.status().spoolBytes, payload.length, "unacknowledged final response must remain spooled")
+  assert.equal(socket.destroyed, false)
+
+  await hub.closeActiveGateway("o2")
+  await waitUntil(() => gateway.status().physicalConnects >= 2 && gateway.status().physicalState === "ready", "closing gateway resume")
+  assert.deepEqual(await withTimeout(response, "resumed final response"), payload)
+  await withTimeout(closed, "resumed CLOSE acknowledgement")
+  await waitUntil(() => hub.status().virtualStreams === 0 && gateway.status().virtualStreams === 0 && client.status().virtualStreams === 0 && gateway.status().spoolBytes === 0, "resumed close cleanup")
+  assert.deepEqual(errors, [])
+})
+
+test("a gateway in paired resume is unavailable for selection and new OPEN", async (t) => {
+  const root = await mkdtemp(join(tmpdir(), "socketduct-reverse-resume-admission-"))
+  t.after(() => rm(root, {recursive: true, force: true}))
+  const certificates = generateTestCertificates(root)
+  let targetAccepts = 0
+  const targetSockets = new Set()
+  const target = createServer((socket) => {
+    targetAccepts += 1
+    targetSockets.add(socket)
+    socket.once("close", () => targetSockets.delete(socket))
+    socket.on("data", (chunk) => socket.write(chunk))
+  })
+  const targetPort = await listen(target)
+  t.after(() => { for (const socket of targetSockets) socket.destroy(); return new Promise((resolve) => target.close(() => resolve(undefined))) })
+  const tokens = new StaticTokenStore([{username: "home", token: "gateway-token"}, {username: "app", token: "client-token"}])
+  const hub = await startReverseGatewayHub({
+    tokens,
+    tls: {certFile: certificates.serverCertFile, keyFile: certificates.serverKeyFile},
+    knownGatewayIds: ["o2"],
+    knownRouteClasses: ["starlink"],
+    authorizeGateway: () => true,
+    authorizeClientGateway: () => true,
+    authorizeClientRouteClass: () => true,
+    resolveOpenGateways: () => ["o2"],
+    authorizeOpen: () => true,
+    resumeWindowMs: 2_000
+  })
+  t.after(() => hub.close())
+  const tls = {caFile: certificates.caCertFile, servername: "localhost"}
+  let gatewayPhysicals = 0
+  /** @type {() => void} */
+  let markResumeHeld = () => {}
+  const resumeHeld = new Promise((resolve) => { markResumeHeld = () => resolve(undefined) })
+  /** @type {() => void} */
+  let releaseResume = () => {}
+  let resumeCaptured = false
+  const gateway = new OutboundReverseGateway({
+    gatewayId: "o2",
+    connectionId: "resume_admission_gateway",
+    hub: {host: "127.0.0.1", port: hub.port, token: "gateway-token"},
+    tls,
+    authorizeTarget: () => true,
+    reconnectDelayMs: 10,
+    maxReconnectDelayMs: 20,
+    createPhysicalSocket: (options) => {
+      gatewayPhysicals += 1
+      const physical = connectTls({host: options.host, port: options.port, ...options.tls})
+      const originalWrite = physical.write.bind(physical)
+      physical.write = /** @type {typeof physical.write} */ ((chunk, ...args) => {
+        if (gatewayPhysicals > 1 && Buffer.isBuffer(chunk) && chunk[5] === V3FrameType.RESUME && !resumeCaptured) {
+          resumeCaptured = true
+          const held = Buffer.from(chunk)
+          releaseResume = () => { Reflect.apply(originalWrite, physical, [held, ...args]) }
+          markResumeHeld()
+          return true
+        }
+        return Reflect.apply(originalWrite, physical, [chunk, ...args])
+      })
+      return physical
+    }
+  })
+  t.after(() => gateway.close())
+  await gateway.start()
+  const retainedClient = new ReverseGatewayConnection({gatewayId: "o2", connectionId: "resume_admission_retained", hub: {host: "127.0.0.1", port: hub.port, token: "client-token"}, tls, reconnectDelayMs: 10, maxReconnectDelayMs: 20})
+  t.after(() => retainedClient.close())
+  await retainedClient.connect()
+  const retained = retainedClient.createConnection({host: "127.0.0.1", port: targetPort})
+  await onceEvent(retained, "connect")
+  const routeClient = new ReverseGatewayConnection({routeClass: "starlink", connectionId: "resume_admission_route", hub: {host: "127.0.0.1", port: hub.port, token: "client-token"}, tls})
+  t.after(() => routeClient.close())
+  await routeClient.connect()
+
+  await hub.closeActiveGateway("o2")
+  await withTimeout(resumeHeld, "gateway RESUME admission hold")
+  assert.equal(hub.status().healthyGateways, 0, "resuming gateway must not be advertised healthy")
+
+  const selectingClient = new ReverseGatewayConnection({gatewayId: "o2", connectionId: "resume_admission_new", hub: {host: "127.0.0.1", port: hub.port, token: "client-token"}, tls, reconnectDelayMs: 50, maxReconnectDelayMs: 50})
+  t.after(() => selectingClient.close())
+  void selectingClient.connect().catch(() => {})
+  await new Promise((resolve) => setTimeout(resolve, 100))
+  assert.equal(selectingClient.status().physicalConnects, 0, "new exact selection must wait until paired resume completes")
+
+  const deniedOpen = routeClient.createConnection({host: "127.0.0.1", port: targetPort})
+  assert.equal((await withTimeout(nextError(deniedOpen), "OPEN while gateway resumes")).code, "GATEWAY_UNAVAILABLE")
+  assert.equal(targetAccepts, 1, "resume-only gateway must not receive a new target OPEN")
+
+  releaseResume()
+  await waitUntil(() => gateway.status().physicalState === "ready" && retainedClient.status().physicalState === "ready", "paired resume after admission hold")
+  const echoed = readBytes(retained, 14)
+  retained.write("retained-alive")
+  assert.equal((await withTimeout(echoed, "retained stream after admission hold")).toString(), "retained-alive")
+  retained.destroy()
+})
+
+test("paired resume preserves unread reverse DATA against the receive window", async (t) => {
+  const root = await mkdtemp(join(tmpdir(), "socketduct-reverse-resume-window-"))
+  t.after(() => rm(root, {recursive: true, force: true}))
+  const certificates = generateTestCertificates(root)
+  /** @type {import('node:net').Socket | undefined} */
+  let targetSocket
+  const targetSockets = new Set()
+  const target = createServer((socket) => {
+    targetSocket = socket
+    targetSockets.add(socket)
+    socket.once("close", () => targetSockets.delete(socket))
+  })
+  const targetPort = await listen(target)
+  t.after(() => { for (const socket of targetSockets) socket.destroy(); return new Promise((resolve) => target.close(() => resolve(undefined))) })
+  const tokens = new StaticTokenStore([{username: "home", token: "gateway-token"}, {username: "app", token: "client-token"}])
+  const hub = await startReverseGatewayHub({tokens, tls: {certFile: certificates.serverCertFile, keyFile: certificates.serverKeyFile}, authorizeGateway: () => true, authorizeClientGateway: () => true, authorizeOpen: () => true, initialWindowBytes: 32, maxFrameBytes: 16, resumeWindowMs: 2_000})
+  t.after(() => hub.close())
+  const tls = {caFile: certificates.caCertFile, servername: "localhost"}
+  const gateway = new OutboundReverseGateway({gatewayId: "o2", connectionId: "resume_window_gateway", hub: {host: "127.0.0.1", port: hub.port, token: "gateway-token"}, tls, spoolDirectory: join(root, "gateway-spool"), authorizeTarget: () => true, initialWindowBytes: 32, maxFrameBytes: 16, reconnectDelayMs: 10, maxReconnectDelayMs: 20})
+  t.after(() => gateway.close())
+  await gateway.start()
+  const client = new ReverseGatewayConnection({gatewayId: "o2", connectionId: "resume_window_client", hub: {host: "127.0.0.1", port: hub.port, token: "client-token"}, tls, spoolDirectory: join(root, "client-spool"), initialWindowBytes: 32, maxFrameBytes: 16, reconnectDelayMs: 10, maxReconnectDelayMs: 20})
+  t.after(() => client.close())
+  await client.connect()
+  const socket = client.createConnection({host: "127.0.0.1", port: targetPort, readableHighWaterMark: 1})
+  await onceEvent(socket, "connect")
+  assert(targetSocket !== undefined)
+
+  targetSocket.write(Buffer.alloc(32, 0x61))
+  await waitUntil(() => socket.readableLength === 32 && gateway.status().spoolBytes === 32, "initial unread window")
+  await hub.closeActiveGateway("o2")
+  await waitUntil(() => gateway.status().physicalConnects >= 2 && gateway.status().physicalState === "ready", "first unread resume")
+  assert.equal(gateway.status().spoolBytes, 32, "resume must not ACK unread VirtualSocket bytes")
+
+  targetSocket.write(Buffer.alloc(32, 0x62))
+  await waitUntil(() => gateway.status().spoolBytes === 64, "second target window retained")
+  await new Promise((resolve) => setTimeout(resolve, 50))
+  assert.equal(socket.readableLength, 32, "resume must not replenish credit for unread bytes")
+  await hub.closeActiveGateway("o2")
+  await waitUntil(() => gateway.status().physicalConnects >= 3 && gateway.status().physicalState === "ready", "second unread resume")
+  assert.equal(socket.readableLength, 32, "repeated resume must preserve the application-consumed boundary")
+  assert.equal(gateway.status().spoolBytes, 64)
+
+  const received = await withTimeout(readBytes(socket, 64), "flow resumes after application consumption")
+  assert.deepEqual(received, Buffer.concat([Buffer.alloc(32, 0x61), Buffer.alloc(32, 0x62)]))
+  const closed = onceEvent(socket, "close")
+  targetSocket.end()
+  await withTimeout(closed, "unread resume cleanup")
+  await waitUntil(() => hub.status().virtualStreams === 0 && gateway.status().virtualStreams === 0 && client.status().virtualStreams === 0 && gateway.status().spoolBytes === 0 && client.status().spoolBytes === 0, "unread resume ownership cleanup")
+})
+
+test("client CLOSE uses the receiver-confirmed sequence immediately after resume", async (t) => {
+  const root = await mkdtemp(join(tmpdir(), "socketduct-reverse-resume-close-sequence-"))
+  t.after(() => rm(root, {recursive: true, force: true}))
+  const certificates = generateTestCertificates(root)
+  const siblingSockets = new Set()
+  const siblingTarget = createServer((socket) => {
+    siblingSockets.add(socket)
+    socket.once("close", () => siblingSockets.delete(socket))
+    socket.on("data", (chunk) => socket.write(chunk))
+  })
+  const siblingPort = await listen(siblingTarget)
+  t.after(() => { for (const socket of siblingSockets) socket.destroy(); return new Promise((resolve) => siblingTarget.close(() => resolve(undefined))) })
+  const tokens = new StaticTokenStore([{username: "home", token: "gateway-token"}, {username: "app", token: "client-token"}])
+  const hub = await startReverseGatewayHub({tokens, tls: {certFile: certificates.serverCertFile, keyFile: certificates.serverKeyFile}, authorizeGateway: () => true, authorizeClientGateway: () => true, authorizeOpen: () => true, initialWindowBytes: 65_536, maxFrameBytes: 65_536, resumeWindowMs: 2_000})
+  t.after(() => hub.close())
+  const tls = {caFile: certificates.caCertFile, servername: "localhost"}
+  /** @type {InterruptibleTarget | undefined} */
+  let closingTarget
+  const gateway = new OutboundReverseGateway({gatewayId: "o2", connectionId: "resume_close_sequence_gateway", hub: {host: "127.0.0.1", port: hub.port, token: "gateway-token"}, tls, authorizeTarget: () => true, initialWindowBytes: 65_536, maxFrameBytes: 65_536, reconnectDelayMs: 40, maxReconnectDelayMs: 40, createTargetConnection: (target) => {
+    if (target.host !== "closing-target") return connectTcp(target)
+    closingTarget = new InterruptibleTarget()
+    return /** @type {import('node:net').Socket} */ (/** @type {unknown} */ (closingTarget))
+  }})
+  t.after(() => gateway.close())
+  await gateway.start()
+  let clientPhysicals = 0
+  /** @type {number | undefined} */
+  let closingStreamId
+  /** @type {() => void} */
+  let markDataHeld = () => {}
+  const dataHeld = new Promise((resolve) => { markDataHeld = () => resolve(undefined) })
+  /** @type {() => void} */
+  let markCloseCoalesced = () => {}
+  const closeCoalesced = new Promise((resolve) => { markCloseCoalesced = () => resolve(undefined) })
+  const client = new ReverseGatewayConnection({
+    gatewayId: "o2",
+    connectionId: "resume_close_sequence_client",
+    hub: {host: "127.0.0.1", port: hub.port, token: "client-token"},
+    tls,
+    initialWindowBytes: 65_536,
+    maxFrameBytes: 65_536,
+    reconnectDelayMs: 10,
+    maxReconnectDelayMs: 20,
+    createPhysicalSocket: (options) => {
+      clientPhysicals += 1
+      const generation = clientPhysicals
+      const physical = connectTls({host: options.host, port: options.port, ...options.tls})
+      const originalWrite = physical.write.bind(physical)
+      physical.write = /** @type {typeof physical.write} */ ((chunk, ...args) => {
+        if (generation === 1 && Buffer.isBuffer(chunk) && chunk[5] === V3FrameType.DATA && chunk.readUInt32BE(8) === closingStreamId) {
+          markDataHeld()
+          return true
+        }
+        return Reflect.apply(originalWrite, physical, [chunk, ...args])
+      })
+      if (generation > 1) {
+        const originalEmit = physical.emit.bind(physical)
+        const decoder = new V3FrameDecoder({maxFrameBytes: 128 * 1024, maxBufferedBytes: 256 * 1024})
+        /** @type {Buffer[]} */
+        let heldInbound = []
+        physical.emit = /** @type {typeof physical.emit} */ ((event, ...args) => {
+          if (event !== "data" || !Buffer.isBuffer(args[0])) return originalEmit(event, ...args)
+          for (const frame of decoder.push(args[0])) {
+            const wire = encodeV3Frame({type: frame.type, streamId: frame.streamId, sequence: frame.sequence, flags: frame.flags}, frame.payload)
+            if (frame.type === V3FrameType.RESUMED) heldInbound.push(wire)
+            else if (heldInbound.length > 0) heldInbound.push(wire)
+            else originalEmit("data", wire)
+            if (heldInbound.length > 0 && frame.type === V3FrameType.CLOSE && frame.streamId === closingStreamId) {
+              const coalesced = Buffer.concat(heldInbound)
+              heldInbound = []
+              originalEmit("data", coalesced)
+              markCloseCoalesced()
+            }
+          }
+          return true
+        })
+      }
+      return physical
+    }
+  })
+  t.after(() => client.close())
+  await client.connect()
+  const closing = client.createConnection({host: "closing-target", port: 2375})
+  closingStreamId = closing.streamId
+  const closingRead = readToEnd(closing)
+  const sibling = client.createConnection({host: "127.0.0.1", port: siblingPort})
+  /** @type {Error[]} */
+  const siblingErrors = []
+  sibling.on("error", (error) => siblingErrors.push(error))
+  await withTimeout(Promise.all([onceEvent(closing, "connect"), onceEvent(sibling, "connect")]), "streams before close sequence resume")
+  closing.write(Buffer.alloc(65_536, 0x63))
+  await withTimeout(dataHeld, "unacknowledged client DATA before resume")
+
+  await hub.closeActiveGateway("o2")
+  await waitUntil(() => gateway.status().physicalState === "reconnecting", "gateway interrupted before target close")
+  assert(closingTarget !== undefined)
+  closingTarget.closeLocally()
+  await withTimeout(closeCoalesced, "RESUMED and target CLOSE delivery")
+  assert.equal((await withTimeout(closingRead, "receiver-confirmed close sequence")).length, 0)
+  const echoed = readBytes(sibling, 13)
+  sibling.write("sibling-alive")
+  assert.equal((await withTimeout(echoed, "sibling after immediate resumed CLOSE")).toString(), "sibling-alive")
+  assert.deepEqual(siblingErrors, [])
+  assert.equal(client.status().physicalConnects, 2, "invalid CLOSE must not recycle the shared client physical")
+  sibling.destroy()
+})
+
+test("a target lost during reverse resume fails the logical stream once with its causal error", async (t) => {
+  const root = await mkdtemp(join(tmpdir(), "socketduct-reverse-resume-target-loss-"))
+  t.after(() => rm(root, {recursive: true, force: true}))
+  const certificates = generateTestCertificates(root)
+  const tokens = new StaticTokenStore([{username: "home", token: "gateway-token"}, {username: "app", token: "client-token"}])
+  const hub = await startReverseGatewayHub({tokens, tls: {certFile: certificates.serverCertFile, keyFile: certificates.serverKeyFile}, authorizeGateway: () => true, authorizeClientGateway: () => true, authorizeOpen: () => true, resumeWindowMs: 1_000})
   t.after(() => hub.close())
   const tls = {caFile: certificates.caCertFile, servername: "localhost"}
-  const gateway = new OutboundReverseGateway({gatewayId: "o2", connectionId: "reconnecting_gateway", hub: {host: "127.0.0.1", port: hub.port, token: "gateway-token"}, tls, authorizeTarget: () => true, reconnectDelayMs: 10, maxReconnectDelayMs: 20})
+  /** @type {InterruptibleTarget | undefined} */
+  let target
+  const gateway = new OutboundReverseGateway({
+    gatewayId: "o2",
+    connectionId: "resume_target_loss_gateway",
+    hub: {host: "127.0.0.1", port: hub.port, token: "gateway-token"},
+    tls,
+    spoolDirectory: join(root, "gateway-spool"),
+    authorizeTarget: () => true,
+    reconnectDelayMs: 80,
+    maxReconnectDelayMs: 80,
+    createTargetConnection: () => {
+      target = new InterruptibleTarget()
+      return /** @type {import('node:net').Socket} */ (/** @type {unknown} */ (target))
+    }
+  })
+  t.after(() => gateway.close())
+  await gateway.start()
+  const client = new ReverseGatewayConnection({gatewayId: "o2", connectionId: "resume_target_loss_client", hub: {host: "127.0.0.1", port: hub.port, token: "client-token"}, tls, spoolDirectory: join(root, "client-spool"), reconnectDelayMs: 10, maxReconnectDelayMs: 20})
+  t.after(() => client.close())
+  await client.connect()
+  const socket = client.createConnection({host: "target", port: 2375})
+  let errors = 0
+  const terminal = new Promise((resolve) => socket.on("error", (error) => { errors += 1; resolve(error) }))
+  await onceEvent(socket, "connect")
+  assert(target !== undefined)
+  await hub.closeActiveGateway("o2")
+  await waitUntil(() => gateway.status().physicalState === "reconnecting", "gateway interruption before target loss")
+  target.fail()
+  const error = /** @type {Error & {code?: string}} */ (await withTimeout(terminal, "target-loss resume error"))
+  assert.equal(error.code, "TARGET_CLOSED")
+  await waitUntil(() => hub.status().virtualStreams === 0 && gateway.status().virtualStreams === 0 && client.status().virtualStreams === 0, "target-loss stream cleanup")
+  assert.equal(errors, 1)
+})
+
+test("an expired reverse resume window fails the logical stream once and releases its target", async (t) => {
+  const root = await mkdtemp(join(tmpdir(), "socketduct-reverse-resume-expiry-"))
+  t.after(() => rm(root, {recursive: true, force: true}))
+  const certificates = generateTestCertificates(root)
+  const tokens = new StaticTokenStore([{username: "home", token: "gateway-token"}, {username: "app", token: "client-token"}])
+  const hub = await startReverseGatewayHub({
+    tokens,
+    tls: {certFile: certificates.serverCertFile, keyFile: certificates.serverKeyFile},
+    knownGatewayIds: ["o2"],
+    knownRouteClasses: ["starlink"],
+    authorizeGateway: () => true,
+    authorizeClientGateway: () => false,
+    authorizeClientRouteClass: () => true,
+    resolveOpenGateways: () => ["o2"],
+    authorizeOpen: () => true,
+    resumeWindowMs: 40
+  })
+  t.after(() => hub.close())
+  const tls = {caFile: certificates.caCertFile, servername: "localhost"}
+  /** @type {InterruptibleTarget[]} */
+  const targets = []
+  const gateway = new OutboundReverseGateway({gatewayId: "o2", connectionId: "resume_expiry_gateway", hub: {host: "127.0.0.1", port: hub.port, token: "gateway-token"}, tls, spoolDirectory: join(root, "gateway-spool"), authorizeTarget: () => true, reconnectDelayMs: 150, maxReconnectDelayMs: 150, createTargetConnection: () => {
+    const target = new InterruptibleTarget()
+    targets.push(target)
+    return /** @type {import('node:net').Socket} */ (/** @type {unknown} */ (target))
+  }})
   t.after(() => gateway.close())
   await gateway.start()
-  const client = new ReverseGatewayConnection({gatewayId: "o2", connectionId: "reconnecting_client", hub: {host: "127.0.0.1", port: hub.port, token: "client-token"}, tls, reconnectDelayMs: 10, maxReconnectDelayMs: 20})
+  const client = new ReverseGatewayConnection({routeClass: "starlink", connectionId: "resume_expiry_client", hub: {host: "127.0.0.1", port: hub.port, token: "client-token"}, tls, spoolDirectory: join(root, "client-spool"), reconnectDelayMs: 5, maxReconnectDelayMs: 10})
   t.after(() => client.close())
   await client.connect()
-  const uncertain = client.createConnection({host: "127.0.0.1", port: targetPort})
-  await onceEvent(uncertain, "connect")
-  const lost = nextError(uncertain)
+  const socket = client.createConnection({host: "target", port: 2375})
+  let errors = 0
+  const terminal = new Promise((resolve) => socket.on("error", (error) => { errors += 1; resolve(error) }))
+  await onceEvent(socket, "connect")
   await hub.closeActiveGateway("o2")
-  assert.equal((await lost).code, "GATEWAY_DISCONNECTED")
-  await waitUntil(() => gateway.status().physicalConnects >= 2 && client.status().physicalConnects >= 2 && gateway.status().physicalState === "ready" && client.status().physicalState === "ready", "gateway and selected member reconnect")
-  await waitUntil(() => targetSockets.size === 0, "uncertain target cleanup")
-  assert.equal(hub.status().virtualStreams, 0)
-  assert.equal(targetSockets.size, 0)
-
-  const clean = client.createConnection({host: "127.0.0.1", port: targetPort})
-  await onceEvent(clean, "connect")
-  const echoed = readBytes(clean, 12)
-  clean.write("clean-retry!")
-  assert.equal((await echoed).toString(), "clean-retry!")
-  assert(gateway.status().physicalConnects >= 2)
-  assert(client.status().physicalConnects >= 2)
-  clean.destroy()
+  const error = /** @type {Error & {code?: string}} */ (await withTimeout(terminal, "expired reverse resume"))
+  assert.equal(error.code, "RESUME_WINDOW_EXPIRED")
+  await waitUntil(() => gateway.status().physicalConnects >= 2 && gateway.status().virtualStreams === 0 && client.status().virtualStreams === 0 && hub.status().virtualStreams === 0, "expired reverse resume cleanup")
+  assert.equal(errors, 1)
+  assert.equal(targets.length, 1)
+  assert.equal(targets[0]?.destroyed, true)
 })
 
 test("reverse mode enforces verified TLS and stream resource bounds", async (t) => {