socketduct 0.1.18 → 0.1.19

package/test/integration/reverse-gateway.test.js

@@ -1,17 +1,21 @@
 /* eslint-disable jsdoc/escape-inline-tags, jsdoc/require-param, jsdoc/require-returns */
 import assert from "node:assert/strict"
-import {mkdtemp, readdir, rm} from "node:fs/promises"
+import {mkdtemp, readFile, readdir, rm, stat, writeFile} from "node:fs/promises"
 import {connect as connectTcp, createServer} from "node:net"
 import {tmpdir} from "node:os"
 import {join} from "node:path"
 import {Duplex} from "node:stream"
 import test from "node:test"
 import {connect as connectTls} from "node:tls"
+import zlib from "node:zlib"
 
 import {StaticTokenStore} from "../../src/auth/static-token-store.js"
 import {MultiplexedSocketductTransportSet} from "../../src/client/multiplexed-transport-set.js"
+import {ZSTD_CODEC} from "../../src/protocol/compression.js"
+import {encodeV3Data, AdaptiveCompressor} from "../../src/protocol/v3-compression.js"
 import {encodeV3Control, encodeV3Frame, V3FrameDecoder, V3FrameType} from "../../src/protocol/v3-frame.js"
 import {OutboundReverseGateway, ReverseGatewayConnection, startReverseGatewayHub} from "../../src/reverse/index.js"
+import {SessionSpool} from "../../src/spool/spool-file.js"
 import {generateTestCertificates} from "../helpers/tls-certificates.js"
 
 /** @param {import('node:net').Server} server - Server. */
@@ -93,6 +97,44 @@ async function waitUntil(predicate, label) {
   }
 }
 
+/**
+ * @param {string} spoolDirectory - Private persistent spool root.
+ * @param {Partial<Record<string, unknown>>} [overrides] - Recovery-state overrides.
+ * @param {Array<{sequence: number, data: Buffer}>} [chunks] - Durable client replay chunks.
+ */
+async function seedReverseClientRecovery(spoolDirectory, overrides = {}, chunks = []) {
+  /** @type {Record<string, unknown>} */
+  const state = {
+    version: 3,
+    role: "reverse-client",
+    connectionId: "recovered_client_01",
+    generation: 41,
+    gatewayId: "o2",
+    streamId: 1,
+    identifier: "recovered_stream_01",
+    name: "durable-reverse-stream",
+    target: {host: "docker-server", port: 2375},
+    state: "open",
+    opened: true,
+    openingSent: true,
+    sendSequence: chunks.at(-1)?.sequence ?? 0,
+    sendAck: 0,
+    receiveSequence: 0,
+    receiveAck: 0,
+    localEnded: false,
+    remoteEnded: false,
+    endSent: false,
+    ...overrides
+  }
+  if (state.gatewayId === undefined) delete state.gatewayId
+  const identifier = /** @type {string} */ (state.identifier)
+  const spool = new SessionSpool({rootDir: spoolDirectory, maxBytes: 1024 * 1024, maxAgeMs: 60_000})
+  await spool.createNew({sessionId: identifier, ownerId: "reverse-client-v3"})
+  for (const chunk of chunks) await spool.appendChunk({sessionId: identifier, ownerId: "reverse-client-v3", ...chunk})
+  await spool.writeRecoveryState(identifier, state)
+  return state
+}
+
 /**
  * @param {{connect(): Promise<void>}} connection - Terminal reverse connection.
  * @param {string} label - Failure context.
@@ -265,6 +307,373 @@ test("reverse client contains a TLS AUTH write race inside the transport", async
   await assert.rejects(connecting, /closed/i)
 })
 
+test("recovered reverse clients require explicit cold durable identity before any physical attempt", async (t) => {
+  const root = await mkdtemp(join(tmpdir(), "socketduct-reverse-recovery-contract-"))
+  t.after(() => rm(root, {recursive: true, force: true}))
+  const spoolDirectory = join(root, "client-spool")
+  await seedReverseClientRecovery(spoolDirectory, {sendAck: 2}, [
+    {sequence: 2, data: Buffer.from("aa")},
+    {sequence: 4, data: Buffer.from("bb")}
+  ])
+  let physicalAttempts = 0
+  /** @type {Record<string, unknown>[]} */
+  const diagnostics = []
+  const base = {
+    gatewayId: "o2",
+    connectionId: "recovered_client_01",
+    hub: {host: "127.0.0.1", port: 3100, token: "client-token"},
+    tls: {servername: "localhost"},
+    spoolDirectory,
+    initialGeneration: 0,
+    logger: {log: (/** @type {string} */ event, /** @type {Record<string, unknown>} */ fields = {}) => diagnostics.push({event, ...fields})},
+    createPhysicalSocket: () => {
+      physicalAttempts += 1
+      return /** @type {import('node:tls').TLSSocket} */ (/** @type {unknown} */ (new AuthWriteSocket()))
+    }
+  }
+
+  const client = new ReverseGatewayConnection(base)
+  t.after(() => client.close())
+  assert.throws(() => client.createConnection({recover: true, identifier: "recovered_stream_01", host: "docker-server", port: 2375}), /must use registerRecoveredConnection/i)
+  assert.throws(() => client.registerRecoveredConnection(/** @type {import('../../src/client/multiplexed-connection.js').VirtualConnectionOptions} */ ({identifier: "recovered_stream_01", host: "docker-server", port: 2375})), /recover: true/i)
+  const recovered = client.registerRecoveredConnection({recover: true, identifier: "recovered_stream_01", sessionName: "durable-reverse-stream", host: "docker-server", port: 2375})
+  assert.equal(recovered.streamId, 1)
+  assert.equal(client.status().physicalState, "configured")
+  assert.equal(client.status().connectionGeneration, 41)
+  assert.equal(client.status().spoolBytes, 2)
+  assert.equal(physicalAttempts, 0)
+
+  const recoveryContents = await readFile(join(spoolDirectory, "recovered_stream_01", "recovery.json"), "utf8")
+  assert.doesNotMatch(recoveryContents, /client-token/)
+  assert.doesNotMatch(JSON.stringify(diagnostics), /client-token/)
+  assert.equal((await stat(spoolDirectory)).mode & 0o777, 0o700)
+  for (const file of ["metadata.json", "recovery.json", "chunks.jsonl", "usage.json"]) {
+    assert.equal((await stat(join(spoolDirectory, "recovered_stream_01", file))).mode & 0o777, 0o600)
+  }
+
+  const {connectionId: _connectionId, ...withoutConnectionOptions} = base
+  void _connectionId
+  const withoutConnectionId = new ReverseGatewayConnection(withoutConnectionOptions)
+  assert.throws(() => withoutConnectionId.registerRecoveredConnection({recover: true, identifier: "recovered_stream_01", host: "docker-server", port: 2375}), /explicit stable connectionId/i)
+  await withoutConnectionId.close()
+  const {spoolDirectory: _spoolDirectory, ...withoutSpoolOptions} = base
+  void _spoolDirectory
+  const withoutSpoolDirectory = new ReverseGatewayConnection(withoutSpoolOptions)
+  assert.throws(() => withoutSpoolDirectory.registerRecoveredConnection({recover: true, identifier: "recovered_stream_01", host: "docker-server", port: 2375}), /explicit persistent spoolDirectory/i)
+  await withoutSpoolDirectory.close()
+
+  const routeSpoolDirectory = join(root, "route-spool")
+  await seedReverseClientRecovery(routeSpoolDirectory, {gatewayId: undefined, routeClass: "starlink", identifier: "recovered_route_stream", streamId: 3})
+  const {gatewayId: _gatewayId, ...routeBase} = base
+  void _gatewayId
+  const routeClient = new ReverseGatewayConnection({...routeBase, routeClass: "starlink", spoolDirectory: routeSpoolDirectory})
+  const recoveredRoute = routeClient.registerRecoveredConnection({recover: true, identifier: "recovered_route_stream", host: "docker-server", port: 2375})
+  assert.equal(recoveredRoute.streamId, 3)
+  assert.equal(physicalAttempts, 0)
+  await routeClient.close()
+
+  const lateRoot = join(root, "late-spool")
+  await seedReverseClientRecovery(lateRoot, {identifier: "recovered_stream_02", streamId: 3})
+  const late = new ReverseGatewayConnection({...base, spoolDirectory: lateRoot})
+  const connecting = late.connect()
+  void connecting.catch(() => {})
+  assert.throws(() => late.registerRecoveredConnection({recover: true, identifier: "recovered_stream_02", host: "docker-server", port: 2375}), /before.*physical connection/i)
+  await late.close()
+})
+
+test("recovered reverse state rejects missing corrupt stale mismatched and unbounded spools before network activity", async (t) => {
+  const root = await mkdtemp(join(tmpdir(), "socketduct-reverse-recovery-invalid-"))
+  t.after(() => rm(root, {recursive: true, force: true}))
+  let physicalAttempts = 0
+  let caseNumber = 0
+
+  /**
+   * @param {string} label - Filesystem-safe case label.
+   * @param {Partial<Record<string, unknown>>} [stateOverrides] - Invalid state mutation.
+   * @param {{chunks?: Array<{sequence: number, data: Buffer}>, maxStreamBufferBytes?: number, register?: Record<string, unknown>, mutate?: (input: {spoolDirectory: string, identifier: string, state: Record<string, unknown>}) => Promise<void>, maxAgeMs?: number}} [caseOptions] - Invalid spool shape.
+   */
+  async function assertInvalidRecovery(label, stateOverrides = {}, caseOptions = {}) {
+    caseNumber += 1
+    const spoolDirectory = join(root, `${caseNumber}-${label}`)
+    const identifier = `invalid_stream_${String(caseNumber).padStart(2, "0")}`
+    const state = await seedReverseClientRecovery(spoolDirectory, {identifier, ...stateOverrides}, caseOptions.chunks)
+    await caseOptions.mutate?.({spoolDirectory, identifier, state})
+    const client = new ReverseGatewayConnection({
+      gatewayId: "o2",
+      connectionId: "recovered_client_01",
+      hub: {host: "127.0.0.1", port: 3100, token: "never-written-token"},
+      tls: {servername: "localhost"},
+      spoolDirectory,
+      initialGeneration: 0,
+      ...(caseOptions.maxAgeMs === undefined ? {} : {maxAgeMs: caseOptions.maxAgeMs}),
+      ...(caseOptions.maxStreamBufferBytes === undefined ? {} : {maxStreamBufferBytes: caseOptions.maxStreamBufferBytes}),
+      createPhysicalSocket: () => { physicalAttempts += 1; return /** @type {import('node:tls').TLSSocket} */ (/** @type {unknown} */ (new AuthWriteSocket())) }
+    })
+    const register = {recover: true, identifier, sessionName: "durable-reverse-stream", host: "docker-server", port: 2375, ...caseOptions.register}
+    assert.throws(() => client.registerRecoveredConnection(/** @type {import('../../src/client/multiplexed-connection.js').VirtualConnectionOptions} */ (register)), /Invalid|exceed|recovery/i, label)
+    assert.equal(client.status().virtualStreams, 0, label)
+    await client.close()
+  }
+
+  await assertInvalidRecovery("wrong-connection", {connectionId: "other_client_01"})
+  await assertInvalidRecovery("wrong-selector", {gatewayId: "other"})
+  await assertInvalidRecovery("ambiguous-selector", {routeClass: "starlink"})
+  await assertInvalidRecovery("wrong-role", {role: "reverse-gateway"})
+  await assertInvalidRecovery("wrong-stream", {streamId: 2})
+  await assertInvalidRecovery("zero-generation", {generation: 0})
+  await assertInvalidRecovery("exhausted-generation", {generation: Number.MAX_SAFE_INTEGER})
+  await assertInvalidRecovery("terminal-lifecycle", {state: "closed"})
+  await assertInvalidRecovery("inconsistent-half-close", {localEnded: true})
+  await assertInvalidRecovery("missing-open", {openingSent: false})
+  await assertInvalidRecovery("end-before-open", {state: "opening", opened: false, localEnded: true, endSent: true})
+  await assertInvalidRecovery("remote-end-before-open", {state: "opening", opened: false, remoteEnded: true})
+  await assertInvalidRecovery("wrong-target", {}, {register: {host: "other-target"}})
+  await assertInvalidRecovery("wrong-name", {}, {register: {sessionName: "other-name"}})
+  await assertInvalidRecovery("wrong-identifier", {}, {mutate: async ({spoolDirectory, identifier, state}) => {
+    await writeFile(join(spoolDirectory, identifier, "recovery.json"), `${JSON.stringify({...state, identifier: "different_stream_01"})}\n`)
+  }})
+  await assertInvalidRecovery("noncontiguous", {sendSequence: 2}, {chunks: [{sequence: 2, data: Buffer.from("x")}]} )
+  await assertInvalidRecovery("oversized", {}, {chunks: [{sequence: 8, data: Buffer.alloc(8)}], maxStreamBufferBytes: 4})
+  await assertInvalidRecovery("wrong-owner", {}, {mutate: async ({spoolDirectory, identifier}) => {
+    const metadataPath = join(spoolDirectory, identifier, "metadata.json")
+    const metadata = JSON.parse(await readFile(metadataPath, "utf8"))
+    await writeFile(metadataPath, `${JSON.stringify({...metadata, ownerId: "reverse-gateway-v3"})}\n`)
+  }})
+  await assertInvalidRecovery("corrupt", {}, {mutate: async ({spoolDirectory, identifier}) => {
+    await writeFile(join(spoolDirectory, identifier, "recovery.json"), "{not-json\n")
+  }})
+  await assertInvalidRecovery("expired", {}, {maxAgeMs: 1, mutate: async ({spoolDirectory, identifier}) => {
+    const metadataPath = join(spoolDirectory, identifier, "metadata.json")
+    const metadata = JSON.parse(await readFile(metadataPath, "utf8"))
+    await writeFile(metadataPath, `${JSON.stringify({...metadata, createdAt: "2000-01-01T00:00:00.000Z"})}\n`)
+  }})
+
+  const routeSpoolDirectory = join(root, "wrong-route-class")
+  await seedReverseClientRecovery(routeSpoolDirectory, {gatewayId: undefined, routeClass: "starlink", identifier: "wrong_route_stream"})
+  const wrongRoute = new ReverseGatewayConnection({routeClass: "zyxel", connectionId: "recovered_client_01", hub: {host: "127.0.0.1", port: 3100, token: "never-written-token"}, tls: {servername: "localhost"}, spoolDirectory: routeSpoolDirectory, createPhysicalSocket: () => { physicalAttempts += 1; return /** @type {import('node:tls').TLSSocket} */ (/** @type {unknown} */ (new AuthWriteSocket())) }})
+  assert.throws(() => wrongRoute.registerRecoveredConnection({recover: true, identifier: "wrong_route_stream", host: "docker-server", port: 2375}), /Invalid.*selector/i)
+  await wrongRoute.close()
+
+  const missing = new ReverseGatewayConnection({gatewayId: "o2", connectionId: "recovered_client_01", hub: {host: "127.0.0.1", port: 3100, token: "never-written-token"}, tls: {servername: "localhost"}, spoolDirectory: join(root, "missing"), createPhysicalSocket: () => { physicalAttempts += 1; return /** @type {import('node:tls').TLSSocket} */ (/** @type {unknown} */ (new AuthWriteSocket())) }})
+  assert.throws(() => missing.registerRecoveredConnection({recover: true, identifier: "missing_stream_01", sessionName: "durable-reverse-stream", host: "docker-server", port: 2375}), /Invalid.*recovery/i)
+  await missing.close()
+  assert.equal(physicalAttempts, 0)
+})
+
+test("recovered reverse stream reports session loss after hub state loss without opening a target", async (t) => {
+  const root = await mkdtemp(join(tmpdir(), "socketduct-reverse-recovery-hub-loss-"))
+  t.after(() => rm(root, {recursive: true, force: true}))
+  const certificates = generateTestCertificates(root)
+  const spoolDirectory = join(root, "client-spool")
+  await seedReverseClientRecovery(spoolDirectory, {generation: 100})
+  const tokens = new StaticTokenStore([{username: "home", token: "gateway-token"}, {username: "app", token: "client-token"}])
+  const hub = await startReverseGatewayHub({tokens, tls: {certFile: certificates.serverCertFile, keyFile: certificates.serverKeyFile}, authorizeGateway: () => true, authorizeClientGateway: () => true, authorizeOpen: () => true})
+  t.after(() => hub.close())
+  const tls = {caFile: certificates.caCertFile, servername: "localhost"}
+  let targetCreates = 0
+  const gateway = new OutboundReverseGateway({gatewayId: "o2", connectionId: "hub_loss_gateway_01", hub: {host: "127.0.0.1", port: hub.port, token: "gateway-token"}, tls, authorizeTarget: () => true, createTargetConnection: () => {
+    targetCreates += 1
+    throw new Error("lost hub state must not recreate a target")
+  }})
+  t.after(() => gateway.close())
+  await gateway.start()
+  const client = new ReverseGatewayConnection({gatewayId: "o2", connectionId: "recovered_client_01", hub: {host: "127.0.0.1", port: hub.port, token: "client-token"}, tls, spoolDirectory, initialGeneration: 0})
+  t.after(() => client.close())
+  const socket = client.registerRecoveredConnection({recover: true, identifier: "recovered_stream_01", sessionName: "durable-reverse-stream", host: "docker-server", port: 2375})
+  const lost = nextError(socket)
+  await client.connect()
+  assert.equal((await withTimeout(lost, "hub-state-loss recovery")).code, "RESUME_SESSION_LOST")
+  assert.equal(targetCreates, 0)
+  assert.equal(gateway.status().targetOpens, 0)
+})
+
+test("replacement reverse client restores a spooled stream without OPEN or a second target", async (t) => {
+  const root = await mkdtemp(join(tmpdir(), "socketduct-reverse-client-replacement-"))
+  t.after(() => rm(root, {recursive: true, force: true}))
+  const certificates = generateTestCertificates(root)
+  const spoolDirectory = join(root, "client-spool")
+  let targetAccepts = 0
+  const targetSockets = new Set()
+  const target = createServer((socket) => {
+    targetAccepts += 1
+    targetSockets.add(socket)
+    socket.once("close", () => targetSockets.delete(socket))
+    socket.on("data", (chunk) => socket.write(Buffer.concat([Buffer.from("target:"), chunk])))
+  })
+  const targetPort = await listen(target)
+  t.after(() => { for (const socket of targetSockets) socket.destroy(); return new Promise((resolve) => target.close(() => resolve(undefined))) })
+  const tokens = new StaticTokenStore([{username: "home", token: "gateway-token"}, {username: "app", token: "client-token"}])
+  const hub = await startReverseGatewayHub({tokens, tls: {certFile: certificates.serverCertFile, keyFile: certificates.serverKeyFile}, authorizeGateway: () => true, authorizeClientGateway: () => true, authorizeOpen: () => true, resumeWindowMs: 2_000})
+  t.after(() => hub.close())
+  const tls = {caFile: certificates.caCertFile, servername: "localhost"}
+  const gateway = new OutboundReverseGateway({gatewayId: "o2", connectionId: "recovered_gateway_01", hub: {host: "127.0.0.1", port: hub.port, token: "gateway-token"}, tls, authorizeTarget: () => true, reconnectDelayMs: 10, maxReconnectDelayMs: 20})
+  t.after(() => gateway.close())
+  await gateway.start()
+
+  const connectionOptions = {gatewayId: "o2", connectionId: "recovered_client_live", hub: {host: "127.0.0.1", port: hub.port, token: "client-token"}, tls, spoolDirectory, reconnectDelayMs: 10, maxReconnectDelayMs: 20}
+  const original = new ReverseGatewayConnection(connectionOptions)
+  await original.connect()
+  const originalSocket = original.createConnection({identifier: "recovered_live_stream", sessionName: "durable-docker-operation", host: "127.0.0.1", port: targetPort})
+  await onceEvent(originalSocket, "connect")
+  const initial = readBytes(originalSocket, 13)
+  originalSocket.write("before")
+  assert.equal((await withTimeout(initial, "initial reverse response")).toString(), "target:before")
+  assert.equal(targetAccepts, 1)
+  const oldGeneration = original.status().connectionGeneration
+  await original.detach()
+  const detachedState = JSON.parse(await readFile(join(spoolDirectory, "recovered_live_stream", "recovery.json"), "utf8"))
+  assert.equal(detachedState.receiveAck, 13, "detach must persist application-consumed target bytes")
+
+  /** @type {number[]} */
+  const outboundTypes = []
+  const replacement = new ReverseGatewayConnection({...connectionOptions, initialGeneration: 0, createPhysicalSocket: (options) => {
+    const physical = connectTls({host: options.host, port: options.port, ...options.tls})
+    const originalWrite = physical.write.bind(physical)
+    physical.write = /** @type {typeof physical.write} */ ((chunk, ...args) => {
+      if (Buffer.isBuffer(chunk) && chunk.length >= 6) outboundTypes.push(/** @type {number} */ (chunk[5]))
+      return Reflect.apply(originalWrite, physical, [chunk, ...args])
+    })
+    return physical
+  }})
+  t.after(() => replacement.close())
+  const recovered = replacement.registerRecoveredConnection({recover: true, identifier: "recovered_live_stream", sessionName: "durable-docker-operation", host: "127.0.0.1", port: targetPort})
+  const connected = onceEvent(recovered, "connect")
+  await replacement.connect()
+  await withTimeout(connected, "recovered reverse VirtualSocket")
+  assert(replacement.status().connectionGeneration > oldGeneration)
+  assert.equal(outboundTypes.includes(V3FrameType.OPEN), false)
+  assert.equal(targetAccepts, 1)
+  const resumed = readBytes(recovered, 12)
+  recovered.write("after")
+  assert.equal((await withTimeout(resumed, "recovered reverse response")).toString(), "target:after")
+  assert.equal(targetAccepts, 1)
+  const recoveredClosed = onceEvent(recovered, "close")
+  recovered.destroy()
+  await recoveredClosed
+  await replacement.close()
+})
+
+test("reverse client detach waits for queued asynchronous DATA and prevents late spool mutation", async (t) => {
+  const root = await mkdtemp(join(tmpdir(), "socketduct-reverse-client-detach-barrier-"))
+  t.after(() => rm(root, {recursive: true, force: true}))
+  const certificates = generateTestCertificates(root)
+  const tokens = new StaticTokenStore([{username: "home", token: "gateway-token"}, {username: "app", token: "client-token"}])
+  const hub = await startReverseGatewayHub({tokens, tls: {certFile: certificates.serverCertFile, keyFile: certificates.serverKeyFile}, authorizeGateway: () => true, authorizeClientGateway: () => true, authorizeOpen: () => true, resumeWindowMs: 2_000})
+  t.after(() => hub.close())
+  const tls = {caFile: certificates.caCertFile, servername: "localhost"}
+  /** @type {HeldWriteTarget[]} */
+  const targets = []
+  const gateway = new OutboundReverseGateway({gatewayId: "o2", connectionId: "detach_barrier_gateway", hub: {host: "127.0.0.1", port: hub.port, token: "gateway-token"}, tls, authorizeTarget: () => true, createTargetConnection: () => {
+    const target = new HeldWriteTarget()
+    targets.push(target)
+    return /** @type {import('node:net').Socket} */ (/** @type {unknown} */ (target))
+  }})
+  t.after(() => gateway.close())
+  t.after(() => { for (const target of targets) target.release() })
+  await gateway.start()
+
+  /** @type {import('node:tls').TLSSocket | undefined} */
+  let physical
+  const spoolDirectory = join(root, "client-spool")
+  const client = new ReverseGatewayConnection({gatewayId: "o2", connectionId: "detach_barrier_client", hub: {host: "127.0.0.1", port: hub.port, token: "client-token"}, tls, spoolDirectory, createPhysicalSocket: (options) => {
+    physical = connectTls({host: options.host, port: options.port, ...options.tls})
+    return physical
+  }})
+  t.after(() => client.close())
+  await client.connect()
+  const socket = client.createConnection({identifier: "detach_barrier_stream", host: "held-target", port: 2375})
+  await onceEvent(socket, "connect")
+  assert(physical !== undefined)
+
+  const payload = Buffer.from("queued-after-detach-".repeat(64))
+  const encoded = await encodeV3Data(payload, ZSTD_CODEC, new AdaptiveCompressor())
+  assert.notEqual(encoded.flags, 0, "fixture DATA must cross the asynchronous decompression boundary")
+  const originalDecompress = zlib.zstdDecompress
+  assert.equal(typeof originalDecompress, "function")
+  /** @type {(() => void) | undefined} */
+  let releaseDecompression
+  /** @type {() => void} */
+  let markDecompressionStarted = () => {}
+  const decompressionStarted = new Promise((resolve) => { markDecompressionStarted = () => resolve(undefined) })
+  /** @type {() => void} */
+  let markDecompressionFinished = () => {}
+  const decompressionFinished = new Promise((resolve) => { markDecompressionFinished = () => resolve(undefined) })
+  zlib.zstdDecompress = /** @type {typeof zlib.zstdDecompress} */ ((input, options, callback) => {
+    markDecompressionStarted()
+    releaseDecompression = () => originalDecompress(input, options, (error, result) => {
+      callback(error, result)
+      markDecompressionFinished()
+    })
+  })
+  t.after(() => {
+    releaseDecompression?.()
+    zlib.zstdDecompress = originalDecompress
+  })
+
+  physical.emit("data", encodeV3Frame({type: V3FrameType.DATA, streamId: socket.streamId, sequence: payload.length, flags: encoded.flags}, encoded.payload))
+  await withTimeout(decompressionStarted, "queued reverse DATA decompression")
+  let detached = false
+  const detaching = client.detach().then(() => { detached = true })
+  await new Promise((resolve) => setImmediate(resolve))
+  const resolvedBeforeQueue = detached
+  releaseDecompression?.()
+  releaseDecompression = undefined
+  await withTimeout(decompressionFinished, "released reverse DATA decompression")
+  await withTimeout(detaching, "reverse client detach barrier")
+  await new Promise((resolve) => setImmediate(resolve))
+
+  assert.equal(resolvedBeforeQueue, false, "detach must not resolve while an old-generation frame handler is queued")
+  assert.equal(socket.readableLength, 0, "a queued old-generation frame must not deliver after detach starts")
+  const recovery = JSON.parse(await readFile(join(spoolDirectory, "detach_barrier_stream", "recovery.json"), "utf8"))
+  assert.equal(recovery.receiveSequence, 0)
+  assert.equal(recovery.receiveAck, 0)
+})
+
+test("replacement gateway submits an empty resume manifest and resets omitted streams once", async (t) => {
+  const root = await mkdtemp(join(tmpdir(), "socketduct-reverse-gateway-replacement-"))
+  t.after(() => rm(root, {recursive: true, force: true}))
+  const certificates = generateTestCertificates(root)
+  let targetAccepts = 0
+  const targetSockets = new Set()
+  const target = createServer((socket) => { targetAccepts += 1; targetSockets.add(socket); socket.once("close", () => targetSockets.delete(socket)) })
+  const targetPort = await listen(target)
+  t.after(() => { for (const socket of targetSockets) socket.destroy(); return new Promise((resolve) => target.close(() => resolve(undefined))) })
+  const tokens = new StaticTokenStore([{username: "home", token: "gateway-token"}, {username: "app", token: "client-token"}])
+  const hub = await startReverseGatewayHub({tokens, tls: {certFile: certificates.serverCertFile, keyFile: certificates.serverKeyFile}, authorizeGateway: () => true, authorizeClientGateway: () => true, authorizeOpen: () => true, resumeWindowMs: 2_000})
+  t.after(() => hub.close())
+  const tls = {caFile: certificates.caCertFile, servername: "localhost"}
+  const gatewayOptions = {gatewayId: "o2", connectionId: "replacement_gateway_01", hub: {host: "127.0.0.1", port: hub.port, token: "gateway-token"}, tls, authorizeTarget: () => true}
+  const originalGateway = new OutboundReverseGateway({...gatewayOptions, reconnectDelayMs: 10_000, maxReconnectDelayMs: 10_000})
+  await originalGateway.start()
+  const client = new ReverseGatewayConnection({gatewayId: "o2", connectionId: "replacement_gateway_client", hub: {host: "127.0.0.1", port: hub.port, token: "client-token"}, tls, reconnectDelayMs: 10, maxReconnectDelayMs: 20})
+  t.after(() => client.close())
+  await client.connect()
+  const socket = client.createConnection({identifier: "replacement_stream_01", host: "127.0.0.1", port: targetPort})
+  let resets = 0
+  const reset = new Promise((resolve) => socket.on("error", (error) => { resets += 1; resolve(error) }))
+  await onceEvent(socket, "connect")
+  assert.equal(targetAccepts, 1)
+  const generation = originalGateway.status().connectionGeneration
+  await hub.closeActiveGateway("o2")
+  await originalGateway.close()
+
+  let replacementTargetCreates = 0
+  const replacement = new OutboundReverseGateway({...gatewayOptions, initialGeneration: generation, createTargetConnection: () => {
+    replacementTargetCreates += 1
+    throw new Error("replacement gateway must not recreate a retained target")
+  }})
+  t.after(() => replacement.close())
+  await replacement.start()
+  const error = /** @type {Error & {code?: string}} */ (await withTimeout(reset, "omitted replacement stream reset"))
+  assert.equal(error.code, "RESUME_SESSION_LOST")
+  await waitUntil(() => hub.status().virtualStreams === 0, "omitted bridge cleanup")
+  await new Promise((resolve) => setImmediate(resolve))
+  assert.equal(resets, 1)
+  assert.equal(replacementTargetCreates, 0)
+  assert.equal(replacement.status().targetOpens, 0)
+  assert.equal(targetAccepts, 1)
+})
+
 class DataThenErrorTarget extends Duplex {
   /** @param {Buffer} payload - Bytes observed before the terminal target error. */
   constructor(payload) {
@@ -1111,6 +1520,17 @@ test("reverse hub rejects bad credentials, unknown gateways, duplicate owners, a
   await replacement.start()
   await waitUntil(() => primary.status().physicalState === "closed", "newer generation fences prior owner")
   assert.equal(hub.status().healthyGateways, 1)
+
+  const primaryClient = new ReverseGatewayConnection({gatewayId: "o2", connectionId: "stable_client_owner", initialGeneration: 100, hub: {host: "127.0.0.1", port: hub.port, token: "client-valid-token"}, tls})
+  t.after(() => primaryClient.close())
+  await primaryClient.connect()
+  const equalClient = new ReverseGatewayConnection({gatewayId: "o2", connectionId: "stable_client_owner", initialGeneration: 100, hub: {host: "127.0.0.1", port: hub.port, token: "client-valid-token"}, tls, continuousReconnect: false})
+  t.after(() => equalClient.close())
+  await assert.rejects(withTimeout(equalClient.connect(), "equal client generation"), (error) => /** @type {{code?: string}} */ (error).code === "STALE_GENERATION")
+  const newerClient = new ReverseGatewayConnection({gatewayId: "o2", connectionId: "stable_client_owner", initialGeneration: 200, hub: {host: "127.0.0.1", port: hub.port, token: "client-valid-token"}, tls})
+  t.after(() => newerClient.close())
+  await newerClient.connect()
+  await waitUntil(() => primaryClient.status().physicalState === "closed", "newer client generation fences prior claimant")
 })
 
 test("hub and gateway both enforce target policy without opening a proxy", async (t) => {

package/test/unit/client/multiplexed-transport-set.test.js

@@ -289,6 +289,17 @@ test("route-class members are first-class reverse transports with route-class al
   assert(connectionIds.every((connectionId) => typeof connectionId === "string"))
   assert.equal(new Set(connectionIds).size, 2)
 
+  assert.throws(() => set.recoverConnection(/** @type {Parameters<MultiplexedSocketductTransportSet['recoverConnection']>[0]} */ (/** @type {unknown} */ ({host: "target.internal", port: 443, identifier: "route_stream_01"}))), /exactly one.*gatewayId.*routeClass/i)
+  assert.throws(() => set.recoverConnection(/** @type {Parameters<MultiplexedSocketductTransportSet['recoverConnection']>[0]} */ (/** @type {unknown} */ ({gatewayId: "starlink", routeClass: "zyxel", host: "target.internal", port: 443, identifier: "route_stream_01"}))), /exactly one.*gatewayId.*routeClass/i)
+  assert.throws(() => set.recoverConnection({routeClass: "missing", host: "target.internal", port: 443, identifier: "route_stream_01"}), /unknown routeClass/i)
+  const recovered = set.recoverConnection({routeClass: "zyxel", host: "target.internal", port: 443, identifier: "route_stream_01", sessionName: "durable-route"})
+  assert.equal(/** @type {{gatewayId?: string}} */ (recovered).gatewayId, undefined)
+  assert.equal(/** @type {{routeClass?: string}} */ (recovered).routeClass, "zyxel")
+  assert.equal(Object.getOwnPropertyDescriptor(recovered, "routeClass")?.writable, false)
+  assert.deepEqual(members.map((member) => member.opens), [0, 1])
+  assert.equal(/** @type {{recover?: unknown}} */ (at(members, 1).recoveryConnectionCalls[0]).recover, true)
+  assert.equal(Object.hasOwn(/** @type {object} */ (at(members, 1).recoveryConnectionCalls[0]), "routeClass"), false)
+
   const started = set.start()
   at(members, 0).ready()
   await started
@@ -845,7 +856,7 @@ test("pending socket end reserves and sends DataView bytes", async () => {
   await set.close()
 })
 
-test("recovery requires an original configured gateway and registration before start", async () => {
+test("recovery requires one original configured selector and registration before start", async () => {
   const {set, members} = fixture()
   assert.throws(() => set.recoverConnection(/** @type {Parameters<MultiplexedSocketductTransportSet['recoverConnection']>[0]} */ (/** @type {unknown} */ ({host: "target.internal", port: 443, identifier: "stable_stream_01"}))), /gatewayId/i)
   assert.throws(() => set.recoverConnection({gatewayId: "unknown", host: "target.internal", port: 443, identifier: "stable_stream_01"}), /unknown gatewayId/i)

package/test/unit/spool/spool-file.test.js

@@ -191,6 +191,139 @@ test("SessionSpool synchronously rejects expired durable recovery state", async
   })
 })
 
+test("SessionSpool loadChunksSync restores only unacknowledged owned chunks", async () => {
+  await withTempDir(async (root) => {
+    const sessionId = "session_recovery_chunks"
+    const spool = new SessionSpool({rootDir: root, maxBytes: 16})
+    await spool.createNew({sessionId, ownerId: "reverse-client-v3"})
+    await spool.appendChunk({sessionId, ownerId: "reverse-client-v3", sequence: 3, data: Buffer.from("old")})
+    await spool.appendChunk({sessionId, ownerId: "reverse-client-v3", sequence: 6, data: Buffer.from("new")})
+
+    const chunks = spool.loadChunksSync(sessionId, "reverse-client-v3", 3, 6)
+
+    assert.deepEqual(chunks.map((chunk) => chunk.sequence), [6])
+    assert.equal(chunks[0]?.data.toString(), "new")
+    assert.equal(spool.getUsedBytes(sessionId), 3)
+    assert.throws(
+      () => spool.loadChunksSync(sessionId, "other-owner", 3, 6),
+      /Invalid Socketduct recovery chunks/
+    )
+  })
+})
+
+test("SessionSpool recovery repairs interruption after chunk append before usage update", async () => {
+  await withTempDir(async (root) => {
+    const sessionId = "session_append_interruption"
+    const spool = new SessionSpool({rootDir: root, maxBytes: 16})
+    await spool.createNew({sessionId, ownerId: "reverse-client-v3"})
+    await spool.appendChunk({sessionId, ownerId: "reverse-client-v3", sequence: 3, data: Buffer.from("one")})
+    const chunksPath = join(root, sessionId, "chunks.jsonl")
+    const committed = await readFile(chunksPath, "utf8")
+    await writeFile(chunksPath, `${committed}${JSON.stringify({sequence: 6, dataBase64: Buffer.from("two").toString("base64")})}\n`)
+
+    const chunks = spool.loadChunksSync(sessionId, "reverse-client-v3", 0, 3)
+
+    assert.deepEqual(chunks.map((chunk) => chunk.sequence), [3])
+    assert.equal(await readFile(chunksPath, "utf8"), committed)
+    assert.deepEqual(JSON.parse(await readFile(join(root, sessionId, "usage.json"), "utf8")), {totalBytes: 3})
+    assert.equal((await stat(chunksPath)).mode & 0o777, 0o600)
+    assert.equal((await stat(join(root, sessionId, "usage.json"))).mode & 0o777, 0o600)
+  })
+})
+
+test("SessionSpool recovery repairs interruption after usage update before recovery commit", async () => {
+  await withTempDir(async (root) => {
+    const sessionId = "session_usage_interruption"
+    const spool = new SessionSpool({rootDir: root, maxBytes: 16})
+    await spool.createNew({sessionId, ownerId: "reverse-client-v3"})
+    await spool.appendChunk({sessionId, ownerId: "reverse-client-v3", sequence: 3, data: Buffer.from("one")})
+    await spool.appendChunk({sessionId, ownerId: "reverse-client-v3", sequence: 6, data: Buffer.from("two")})
+
+    const chunks = spool.loadChunksSync(sessionId, "reverse-client-v3", 0, 3)
+
+    assert.deepEqual(chunks.map((chunk) => chunk.sequence), [3])
+    assert.deepEqual(JSON.parse(await readFile(join(root, sessionId, "usage.json"), "utf8")), {totalBytes: 3})
+  })
+})
+
+test("SessionSpool recovery repairs interruption after acknowledged chunk truncation", async () => {
+  await withTempDir(async (root) => {
+    const sessionId = "session_truncate_interruption"
+    const spool = new SessionSpool({rootDir: root, maxBytes: 16})
+    await spool.createNew({sessionId, ownerId: "reverse-client-v3"})
+    await spool.appendChunk({sessionId, ownerId: "reverse-client-v3", sequence: 3, data: Buffer.from("one")})
+    await spool.appendChunk({sessionId, ownerId: "reverse-client-v3", sequence: 6, data: Buffer.from("two")})
+    const chunksPath = join(root, sessionId, "chunks.jsonl")
+    await writeFile(chunksPath, `${JSON.stringify({sequence: 6, dataBase64: Buffer.from("two").toString("base64")})}\n`)
+
+    const chunks = spool.loadChunksSync(sessionId, "reverse-client-v3", 3, 6)
+
+    assert.deepEqual(chunks.map((chunk) => chunk.sequence), [6])
+    assert.deepEqual(JSON.parse(await readFile(join(root, sessionId, "usage.json"), "utf8")), {totalBytes: 3})
+  })
+})
+
+test("SessionSpool recovery repairs a partial trailing JSONL record after committed chunks", async () => {
+  await withTempDir(async (root) => {
+    const sessionId = "session_partial_interruption"
+    const spool = new SessionSpool({rootDir: root, maxBytes: 16})
+    await spool.createNew({sessionId, ownerId: "reverse-client-v3"})
+    await spool.appendChunk({sessionId, ownerId: "reverse-client-v3", sequence: 3, data: Buffer.from("one")})
+    await spool.appendChunk({sessionId, ownerId: "reverse-client-v3", sequence: 6, data: Buffer.from("two")})
+    const chunksPath = join(root, sessionId, "chunks.jsonl")
+    const committed = await readFile(chunksPath, "utf8")
+    await writeFile(chunksPath, `${committed}{"sequence":9,"dataBase64":"thr`)
+
+    const chunks = spool.loadChunksSync(sessionId, "reverse-client-v3", 0, 6)
+
+    assert.deepEqual(chunks.map((chunk) => chunk.sequence), [3, 6])
+    assert.equal(await readFile(chunksPath, "utf8"), committed)
+  })
+})
+
+test("SessionSpool recovery rejects corruption before or outside a partial uncommitted tail", async () => {
+  await withTempDir(async (root) => {
+    const sessionId = "session_committed_corruption"
+    const spool = new SessionSpool({rootDir: root, maxBytes: 16})
+    await spool.createNew({sessionId, ownerId: "reverse-client-v3"})
+    await spool.appendChunk({sessionId, ownerId: "reverse-client-v3", sequence: 3, data: Buffer.from("one")})
+    const chunksPath = join(root, sessionId, "chunks.jsonl")
+    await writeFile(chunksPath, `${await readFile(chunksPath, "utf8")}{"sequence":6,"dataBase64":"tw`)
+
+    assert.throws(
+      () => spool.loadChunksSync(sessionId, "reverse-client-v3", 0, 6),
+      /Invalid Socketduct recovery chunks/
+    )
+  })
+
+  await withTempDir(async (root) => {
+    const sessionId = "session_tail_corruption"
+    const spool = new SessionSpool({rootDir: root, maxBytes: 16})
+    await spool.createNew({sessionId, ownerId: "reverse-client-v3"})
+    await spool.appendChunk({sessionId, ownerId: "reverse-client-v3", sequence: 3, data: Buffer.from("one")})
+    const chunksPath = join(root, sessionId, "chunks.jsonl")
+    await writeFile(chunksPath, `${await readFile(chunksPath, "utf8")}{not-json}\n`)
+
+    assert.throws(
+      () => spool.loadChunksSync(sessionId, "reverse-client-v3", 0, 3),
+      /Invalid Socketduct recovery chunks/
+    )
+  })
+
+  await withTempDir(async (root) => {
+    const sessionId = "session_usage_corruption"
+    const spool = new SessionSpool({rootDir: root, maxBytes: 16})
+    await spool.createNew({sessionId, ownerId: "reverse-client-v3"})
+    await spool.appendChunk({sessionId, ownerId: "reverse-client-v3", sequence: 3, data: Buffer.from("one")})
+    await writeFile(join(root, sessionId, "usage.json"), "{not-json\n")
+
+    assert.throws(
+      () => spool.loadChunksSync(sessionId, "reverse-client-v3", 0, 3),
+      /Invalid Socketduct recovery chunks/
+    )
+  })
+})
+
 test("SessionSpool cleans expired abandoned entries so their stable IDs can be reused", async () => {
   await withTempDir(async (root) => {
     let now = 1_000_000