socketduct 0.1.18 → 0.1.20

package/test/integration/reverse-gateway.test.js

@@ -1,17 +1,21 @@
 /* eslint-disable jsdoc/escape-inline-tags, jsdoc/require-param, jsdoc/require-returns */
 import assert from "node:assert/strict"
-import {mkdtemp, readdir, rm} from "node:fs/promises"
+import {mkdtemp, readFile, readdir, rm, stat, writeFile} from "node:fs/promises"
 import {connect as connectTcp, createServer} from "node:net"
 import {tmpdir} from "node:os"
 import {join} from "node:path"
 import {Duplex} from "node:stream"
 import test from "node:test"
 import {connect as connectTls} from "node:tls"
+import zlib from "node:zlib"
 
 import {StaticTokenStore} from "../../src/auth/static-token-store.js"
 import {MultiplexedSocketductTransportSet} from "../../src/client/multiplexed-transport-set.js"
+import {ZSTD_CODEC} from "../../src/protocol/compression.js"
+import {encodeV3Data, AdaptiveCompressor} from "../../src/protocol/v3-compression.js"
 import {encodeV3Control, encodeV3Frame, V3FrameDecoder, V3FrameType} from "../../src/protocol/v3-frame.js"
 import {OutboundReverseGateway, ReverseGatewayConnection, startReverseGatewayHub} from "../../src/reverse/index.js"
+import {SessionSpool} from "../../src/spool/spool-file.js"
 import {generateTestCertificates} from "../helpers/tls-certificates.js"
 
 /** @param {import('node:net').Server} server - Server. */
@@ -93,6 +97,44 @@ async function waitUntil(predicate, label) {
   }
 }
 
+/**
+ * @param {string} spoolDirectory - Private persistent spool root.
+ * @param {Partial<Record<string, unknown>>} [overrides] - Recovery-state overrides.
+ * @param {Array<{sequence: number, data: Buffer}>} [chunks] - Durable client replay chunks.
+ */
+async function seedReverseClientRecovery(spoolDirectory, overrides = {}, chunks = []) {
+  /** @type {Record<string, unknown>} */
+  const state = {
+    version: 3,
+    role: "reverse-client",
+    connectionId: "recovered_client_01",
+    generation: 41,
+    gatewayId: "o2",
+    streamId: 1,
+    identifier: "recovered_stream_01",
+    name: "durable-reverse-stream",
+    target: {host: "docker-server", port: 2375},
+    state: "open",
+    opened: true,
+    openingSent: true,
+    sendSequence: chunks.at(-1)?.sequence ?? 0,
+    sendAck: 0,
+    receiveSequence: 0,
+    receiveAck: 0,
+    localEnded: false,
+    remoteEnded: false,
+    endSent: false,
+    ...overrides
+  }
+  if (state.gatewayId === undefined) delete state.gatewayId
+  const identifier = /** @type {string} */ (state.identifier)
+  const spool = new SessionSpool({rootDir: spoolDirectory, maxBytes: 1024 * 1024, maxAgeMs: 60_000})
+  await spool.createNew({sessionId: identifier, ownerId: "reverse-client-v3"})
+  for (const chunk of chunks) await spool.appendChunk({sessionId: identifier, ownerId: "reverse-client-v3", ...chunk})
+  await spool.writeRecoveryState(identifier, state)
+  return state
+}
+
 /**
  * @param {{connect(): Promise<void>}} connection - Terminal reverse connection.
  * @param {string} label - Failure context.
@@ -265,6 +307,412 @@ test("reverse client contains a TLS AUTH write race inside the transport", async
   await assert.rejects(connecting, /closed/i)
 })
 
+test("fatal hub GOAWAY preserves validated attribution on the public client error", async (t) => {
+  /** @type {AuthWriteSocket | undefined} */
+  let physical
+  const client = new ReverseGatewayConnection({
+    gatewayId: "o2",
+    connectionId: "fatal_goaway_attribution_client",
+    hub: {host: "127.0.0.1", port: 3100, token: "client-token"},
+    tls: {servername: "localhost"},
+    continuousReconnect: false,
+    createPhysicalSocket: () => {
+      physical = new AuthWriteSocket()
+      return /** @type {import('node:tls').TLSSocket} */ (/** @type {unknown} */ (physical))
+    }
+  })
+  t.after(() => client.close())
+  const connecting = client.connect()
+  physical?.emit("secureConnect")
+  physical?.emit("data", encodeV3Frame({type: V3FrameType.GOAWAY, streamId: 0, sequence: 0}, encodeV3Control({
+    code: "PROTOCOL_VIOLATION",
+    disconnectIncidentId: "AAAAAAAAAAAAAAAAAAAAAA",
+    disconnectReason: "protocol_violation",
+    incidentOrigin: "reverse_hub",
+    physicalRole: "client",
+    clientConnectionGeneration: 1,
+    fanOutCount: 0,
+    resumeSuccessCount: 0,
+    terminalStreamCount: 0
+  })))
+  /** @type {Error & Record<string, unknown> | undefined} */
+  let rejection
+  await assert.rejects(withTimeout(connecting, "fatal attributed GOAWAY"), (error) => {
+    rejection = /** @type {Error & Record<string, unknown>} */ (error)
+    return rejection.code === "PROTOCOL_VIOLATION"
+  })
+  assert.equal(rejection?.disconnectIncidentId, "AAAAAAAAAAAAAAAAAAAAAA")
+  assert.equal(rejection?.disconnectReason, "protocol_violation")
+  assert.equal(rejection?.incidentOrigin, "reverse_hub")
+})
+
+test("recovered reverse clients require explicit cold durable identity before any physical attempt", async (t) => {
+  const root = await mkdtemp(join(tmpdir(), "socketduct-reverse-recovery-contract-"))
+  t.after(() => rm(root, {recursive: true, force: true}))
+  const spoolDirectory = join(root, "client-spool")
+  await seedReverseClientRecovery(spoolDirectory, {sendAck: 2}, [
+    {sequence: 2, data: Buffer.from("aa")},
+    {sequence: 4, data: Buffer.from("bb")}
+  ])
+  let physicalAttempts = 0
+  /** @type {Record<string, unknown>[]} */
+  const diagnostics = []
+  const base = {
+    gatewayId: "o2",
+    connectionId: "recovered_client_01",
+    hub: {host: "127.0.0.1", port: 3100, token: "client-token"},
+    tls: {servername: "localhost"},
+    spoolDirectory,
+    initialGeneration: 0,
+    logger: {log: (/** @type {string} */ event, /** @type {Record<string, unknown>} */ fields = {}) => diagnostics.push({event, ...fields})},
+    createPhysicalSocket: () => {
+      physicalAttempts += 1
+      return /** @type {import('node:tls').TLSSocket} */ (/** @type {unknown} */ (new AuthWriteSocket()))
+    }
+  }
+
+  const client = new ReverseGatewayConnection(base)
+  t.after(() => client.close())
+  assert.throws(() => client.createConnection({recover: true, identifier: "recovered_stream_01", host: "docker-server", port: 2375}), /must use registerRecoveredConnection/i)
+  assert.throws(() => client.registerRecoveredConnection(/** @type {import('../../src/client/multiplexed-connection.js').VirtualConnectionOptions} */ ({identifier: "recovered_stream_01", host: "docker-server", port: 2375})), /recover: true/i)
+  const recovered = client.registerRecoveredConnection({recover: true, identifier: "recovered_stream_01", sessionName: "durable-reverse-stream", host: "docker-server", port: 2375})
+  assert.equal(recovered.streamId, 1)
+  assert.equal(client.status().physicalState, "configured")
+  assert.equal(client.status().connectionGeneration, 41)
+  assert.equal(client.status().spoolBytes, 2)
+  assert.equal(physicalAttempts, 0)
+
+  const recoveryContents = await readFile(join(spoolDirectory, "recovered_stream_01", "recovery.json"), "utf8")
+  assert.doesNotMatch(recoveryContents, /client-token/)
+  assert.doesNotMatch(JSON.stringify(diagnostics), /client-token/)
+  assert.equal((await stat(spoolDirectory)).mode & 0o777, 0o700)
+  for (const file of ["metadata.json", "recovery.json", "chunks.jsonl", "usage.json"]) {
+    assert.equal((await stat(join(spoolDirectory, "recovered_stream_01", file))).mode & 0o777, 0o600)
+  }
+
+  const {connectionId: _connectionId, ...withoutConnectionOptions} = base
+  void _connectionId
+  const withoutConnectionId = new ReverseGatewayConnection(withoutConnectionOptions)
+  assert.throws(() => withoutConnectionId.registerRecoveredConnection({recover: true, identifier: "recovered_stream_01", host: "docker-server", port: 2375}), /explicit stable connectionId/i)
+  await withoutConnectionId.close()
+  const {spoolDirectory: _spoolDirectory, ...withoutSpoolOptions} = base
+  void _spoolDirectory
+  const withoutSpoolDirectory = new ReverseGatewayConnection(withoutSpoolOptions)
+  assert.throws(() => withoutSpoolDirectory.registerRecoveredConnection({recover: true, identifier: "recovered_stream_01", host: "docker-server", port: 2375}), /explicit persistent spoolDirectory/i)
+  await withoutSpoolDirectory.close()
+
+  const routeSpoolDirectory = join(root, "route-spool")
+  await seedReverseClientRecovery(routeSpoolDirectory, {gatewayId: undefined, routeClass: "starlink", identifier: "recovered_route_stream", streamId: 3})
+  const {gatewayId: _gatewayId, ...routeBase} = base
+  void _gatewayId
+  const routeClient = new ReverseGatewayConnection({...routeBase, routeClass: "starlink", spoolDirectory: routeSpoolDirectory})
+  const recoveredRoute = routeClient.registerRecoveredConnection({recover: true, identifier: "recovered_route_stream", host: "docker-server", port: 2375})
+  assert.equal(recoveredRoute.streamId, 3)
+  assert.equal(physicalAttempts, 0)
+  await routeClient.close()
+
+  const lateRoot = join(root, "late-spool")
+  await seedReverseClientRecovery(lateRoot, {identifier: "recovered_stream_02", streamId: 3})
+  const late = new ReverseGatewayConnection({...base, spoolDirectory: lateRoot})
+  const connecting = late.connect()
+  void connecting.catch(() => {})
+  assert.throws(() => late.registerRecoveredConnection({recover: true, identifier: "recovered_stream_02", host: "docker-server", port: 2375}), /before.*physical connection/i)
+  await late.close()
+})
+
+test("recovered reverse state rejects missing corrupt stale mismatched and unbounded spools before network activity", async (t) => {
+  const root = await mkdtemp(join(tmpdir(), "socketduct-reverse-recovery-invalid-"))
+  t.after(() => rm(root, {recursive: true, force: true}))
+  let physicalAttempts = 0
+  let caseNumber = 0
+
+  /**
+   * @param {string} label - Filesystem-safe case label.
+   * @param {Partial<Record<string, unknown>>} [stateOverrides] - Invalid state mutation.
+   * @param {{chunks?: Array<{sequence: number, data: Buffer}>, maxStreamBufferBytes?: number, register?: Record<string, unknown>, mutate?: (input: {spoolDirectory: string, identifier: string, state: Record<string, unknown>}) => Promise<void>, maxAgeMs?: number}} [caseOptions] - Invalid spool shape.
+   */
+  async function assertInvalidRecovery(label, stateOverrides = {}, caseOptions = {}) {
+    caseNumber += 1
+    const spoolDirectory = join(root, `${caseNumber}-${label}`)
+    const identifier = `invalid_stream_${String(caseNumber).padStart(2, "0")}`
+    const state = await seedReverseClientRecovery(spoolDirectory, {identifier, ...stateOverrides}, caseOptions.chunks)
+    await caseOptions.mutate?.({spoolDirectory, identifier, state})
+    const client = new ReverseGatewayConnection({
+      gatewayId: "o2",
+      connectionId: "recovered_client_01",
+      hub: {host: "127.0.0.1", port: 3100, token: "never-written-token"},
+      tls: {servername: "localhost"},
+      spoolDirectory,
+      initialGeneration: 0,
+      ...(caseOptions.maxAgeMs === undefined ? {} : {maxAgeMs: caseOptions.maxAgeMs}),
+      ...(caseOptions.maxStreamBufferBytes === undefined ? {} : {maxStreamBufferBytes: caseOptions.maxStreamBufferBytes}),
+      createPhysicalSocket: () => { physicalAttempts += 1; return /** @type {import('node:tls').TLSSocket} */ (/** @type {unknown} */ (new AuthWriteSocket())) }
+    })
+    const register = {recover: true, identifier, sessionName: "durable-reverse-stream", host: "docker-server", port: 2375, ...caseOptions.register}
+    assert.throws(() => client.registerRecoveredConnection(/** @type {import('../../src/client/multiplexed-connection.js').VirtualConnectionOptions} */ (register)), /Invalid|exceed|recovery/i, label)
+    assert.equal(client.status().virtualStreams, 0, label)
+    await client.close()
+  }
+
+  await assertInvalidRecovery("wrong-connection", {connectionId: "other_client_01"})
+  await assertInvalidRecovery("wrong-selector", {gatewayId: "other"})
+  await assertInvalidRecovery("ambiguous-selector", {routeClass: "starlink"})
+  await assertInvalidRecovery("wrong-role", {role: "reverse-gateway"})
+  await assertInvalidRecovery("wrong-stream", {streamId: 2})
+  await assertInvalidRecovery("zero-generation", {generation: 0})
+  await assertInvalidRecovery("exhausted-generation", {generation: Number.MAX_SAFE_INTEGER})
+  await assertInvalidRecovery("terminal-lifecycle", {state: "closed"})
+  await assertInvalidRecovery("inconsistent-half-close", {localEnded: true})
+  await assertInvalidRecovery("missing-open", {openingSent: false})
+  await assertInvalidRecovery("end-before-open", {state: "opening", opened: false, localEnded: true, endSent: true})
+  await assertInvalidRecovery("remote-end-before-open", {state: "opening", opened: false, remoteEnded: true})
+  await assertInvalidRecovery("wrong-target", {}, {register: {host: "other-target"}})
+  await assertInvalidRecovery("wrong-name", {}, {register: {sessionName: "other-name"}})
+  await assertInvalidRecovery("wrong-identifier", {}, {mutate: async ({spoolDirectory, identifier, state}) => {
+    await writeFile(join(spoolDirectory, identifier, "recovery.json"), `${JSON.stringify({...state, identifier: "different_stream_01"})}\n`)
+  }})
+  await assertInvalidRecovery("noncontiguous", {sendSequence: 2}, {chunks: [{sequence: 2, data: Buffer.from("x")}]} )
+  await assertInvalidRecovery("oversized", {}, {chunks: [{sequence: 8, data: Buffer.alloc(8)}], maxStreamBufferBytes: 4})
+  await assertInvalidRecovery("wrong-owner", {}, {mutate: async ({spoolDirectory, identifier}) => {
+    const metadataPath = join(spoolDirectory, identifier, "metadata.json")
+    const metadata = JSON.parse(await readFile(metadataPath, "utf8"))
+    await writeFile(metadataPath, `${JSON.stringify({...metadata, ownerId: "reverse-gateway-v3"})}\n`)
+  }})
+  await assertInvalidRecovery("corrupt", {}, {mutate: async ({spoolDirectory, identifier}) => {
+    await writeFile(join(spoolDirectory, identifier, "recovery.json"), "{not-json\n")
+  }})
+  await assertInvalidRecovery("expired", {}, {maxAgeMs: 1, mutate: async ({spoolDirectory, identifier}) => {
+    const metadataPath = join(spoolDirectory, identifier, "metadata.json")
+    const metadata = JSON.parse(await readFile(metadataPath, "utf8"))
+    await writeFile(metadataPath, `${JSON.stringify({...metadata, createdAt: "2000-01-01T00:00:00.000Z"})}\n`)
+  }})
+
+  const routeSpoolDirectory = join(root, "wrong-route-class")
+  await seedReverseClientRecovery(routeSpoolDirectory, {gatewayId: undefined, routeClass: "starlink", identifier: "wrong_route_stream"})
+  const wrongRoute = new ReverseGatewayConnection({routeClass: "zyxel", connectionId: "recovered_client_01", hub: {host: "127.0.0.1", port: 3100, token: "never-written-token"}, tls: {servername: "localhost"}, spoolDirectory: routeSpoolDirectory, createPhysicalSocket: () => { physicalAttempts += 1; return /** @type {import('node:tls').TLSSocket} */ (/** @type {unknown} */ (new AuthWriteSocket())) }})
+  assert.throws(() => wrongRoute.registerRecoveredConnection({recover: true, identifier: "wrong_route_stream", host: "docker-server", port: 2375}), /Invalid.*selector/i)
+  await wrongRoute.close()
+
+  const missing = new ReverseGatewayConnection({gatewayId: "o2", connectionId: "recovered_client_01", hub: {host: "127.0.0.1", port: 3100, token: "never-written-token"}, tls: {servername: "localhost"}, spoolDirectory: join(root, "missing"), createPhysicalSocket: () => { physicalAttempts += 1; return /** @type {import('node:tls').TLSSocket} */ (/** @type {unknown} */ (new AuthWriteSocket())) }})
+  assert.throws(() => missing.registerRecoveredConnection({recover: true, identifier: "missing_stream_01", sessionName: "durable-reverse-stream", host: "docker-server", port: 2375}), /Invalid.*recovery/i)
+  await missing.close()
+  assert.equal(physicalAttempts, 0)
+})
+
+test("recovered reverse stream reports session loss after hub state loss without opening a target", async (t) => {
+  const root = await mkdtemp(join(tmpdir(), "socketduct-reverse-recovery-hub-loss-"))
+  t.after(() => rm(root, {recursive: true, force: true}))
+  const certificates = generateTestCertificates(root)
+  const spoolDirectory = join(root, "client-spool")
+  await seedReverseClientRecovery(spoolDirectory, {generation: 100})
+  const tokens = new StaticTokenStore([{username: "home", token: "gateway-token"}, {username: "app", token: "client-token"}])
+  const hub = await startReverseGatewayHub({tokens, tls: {certFile: certificates.serverCertFile, keyFile: certificates.serverKeyFile}, authorizeGateway: () => true, authorizeClientGateway: () => true, authorizeOpen: () => true})
+  t.after(() => hub.close())
+  const tls = {caFile: certificates.caCertFile, servername: "localhost"}
+  let targetCreates = 0
+  const gateway = new OutboundReverseGateway({gatewayId: "o2", connectionId: "hub_loss_gateway_01", hub: {host: "127.0.0.1", port: hub.port, token: "gateway-token"}, tls, authorizeTarget: () => true, createTargetConnection: () => {
+    targetCreates += 1
+    throw new Error("lost hub state must not recreate a target")
+  }})
+  t.after(() => gateway.close())
+  await gateway.start()
+  const client = new ReverseGatewayConnection({gatewayId: "o2", connectionId: "recovered_client_01", hub: {host: "127.0.0.1", port: hub.port, token: "client-token"}, tls, spoolDirectory, initialGeneration: 0})
+  t.after(() => client.close())
+  const socket = client.registerRecoveredConnection({recover: true, identifier: "recovered_stream_01", sessionName: "durable-reverse-stream", host: "docker-server", port: 2375})
+  const lost = nextError(socket)
+  await client.connect()
+  assert.equal((await withTimeout(lost, "hub-state-loss recovery")).code, "RESUME_SESSION_LOST")
+  assert.equal(targetCreates, 0)
+  assert.equal(gateway.status().targetOpens, 0)
+})
+
+test("replacement reverse client restores a spooled stream without OPEN or a second target", async (t) => {
+  const root = await mkdtemp(join(tmpdir(), "socketduct-reverse-client-replacement-"))
+  t.after(() => rm(root, {recursive: true, force: true}))
+  const certificates = generateTestCertificates(root)
+  const spoolDirectory = join(root, "client-spool")
+  let targetAccepts = 0
+  const targetSockets = new Set()
+  const target = createServer((socket) => {
+    targetAccepts += 1
+    targetSockets.add(socket)
+    socket.once("close", () => targetSockets.delete(socket))
+    socket.on("data", (chunk) => socket.write(Buffer.concat([Buffer.from("target:"), chunk])))
+  })
+  const targetPort = await listen(target)
+  t.after(() => { for (const socket of targetSockets) socket.destroy(); return new Promise((resolve) => target.close(() => resolve(undefined))) })
+  const tokens = new StaticTokenStore([{username: "home", token: "gateway-token"}, {username: "app", token: "client-token"}])
+  const hub = await startReverseGatewayHub({tokens, tls: {certFile: certificates.serverCertFile, keyFile: certificates.serverKeyFile}, authorizeGateway: () => true, authorizeClientGateway: () => true, authorizeOpen: () => true, resumeWindowMs: 2_000})
+  t.after(() => hub.close())
+  const tls = {caFile: certificates.caCertFile, servername: "localhost"}
+  const gateway = new OutboundReverseGateway({gatewayId: "o2", connectionId: "recovered_gateway_01", hub: {host: "127.0.0.1", port: hub.port, token: "gateway-token"}, tls, authorizeTarget: () => true, reconnectDelayMs: 10, maxReconnectDelayMs: 20})
+  t.after(() => gateway.close())
+  await gateway.start()
+
+  const connectionOptions = {gatewayId: "o2", connectionId: "recovered_client_live", hub: {host: "127.0.0.1", port: hub.port, token: "client-token"}, tls, spoolDirectory, reconnectDelayMs: 10, maxReconnectDelayMs: 20}
+  const original = new ReverseGatewayConnection(connectionOptions)
+  await original.connect()
+  const originalSocket = original.createConnection({identifier: "recovered_live_stream", sessionName: "durable-docker-operation", host: "127.0.0.1", port: targetPort})
+  await onceEvent(originalSocket, "connect")
+  const initial = readBytes(originalSocket, 13)
+  originalSocket.write("before")
+  assert.equal((await withTimeout(initial, "initial reverse response")).toString(), "target:before")
+  assert.equal(targetAccepts, 1)
+  const oldGeneration = original.status().connectionGeneration
+  await original.detach()
+  const detachedState = JSON.parse(await readFile(join(spoolDirectory, "recovered_live_stream", "recovery.json"), "utf8"))
+  assert.equal(detachedState.receiveAck, 13, "detach must persist application-consumed target bytes")
+
+  /** @type {number[]} */
+  const outboundTypes = []
+  const replacement = new ReverseGatewayConnection({...connectionOptions, initialGeneration: 0, createPhysicalSocket: (options) => {
+    const physical = connectTls({host: options.host, port: options.port, ...options.tls})
+    const originalWrite = physical.write.bind(physical)
+    physical.write = /** @type {typeof physical.write} */ ((chunk, ...args) => {
+      if (Buffer.isBuffer(chunk) && chunk.length >= 6) outboundTypes.push(/** @type {number} */ (chunk[5]))
+      return Reflect.apply(originalWrite, physical, [chunk, ...args])
+    })
+    return physical
+  }})
+  t.after(() => replacement.close())
+  const recovered = replacement.registerRecoveredConnection({recover: true, identifier: "recovered_live_stream", sessionName: "durable-docker-operation", host: "127.0.0.1", port: targetPort})
+  const connected = onceEvent(recovered, "connect")
+  await replacement.connect()
+  await withTimeout(connected, "recovered reverse VirtualSocket")
+  assert(replacement.status().connectionGeneration > oldGeneration)
+  assert.equal(outboundTypes.includes(V3FrameType.OPEN), false)
+  assert.equal(targetAccepts, 1)
+  const resumed = readBytes(recovered, 12)
+  recovered.write("after")
+  assert.equal((await withTimeout(resumed, "recovered reverse response")).toString(), "target:after")
+  assert.equal(targetAccepts, 1)
+  const recoveredClosed = onceEvent(recovered, "close")
+  recovered.destroy()
+  await recoveredClosed
+  await replacement.close()
+})
+
+test("reverse client detach waits for queued asynchronous DATA and prevents late spool mutation", async (t) => {
+  const root = await mkdtemp(join(tmpdir(), "socketduct-reverse-client-detach-barrier-"))
+  t.after(() => rm(root, {recursive: true, force: true}))
+  const certificates = generateTestCertificates(root)
+  const tokens = new StaticTokenStore([{username: "home", token: "gateway-token"}, {username: "app", token: "client-token"}])
+  const hub = await startReverseGatewayHub({tokens, tls: {certFile: certificates.serverCertFile, keyFile: certificates.serverKeyFile}, authorizeGateway: () => true, authorizeClientGateway: () => true, authorizeOpen: () => true, resumeWindowMs: 2_000})
+  t.after(() => hub.close())
+  const tls = {caFile: certificates.caCertFile, servername: "localhost"}
+  /** @type {HeldWriteTarget[]} */
+  const targets = []
+  const gateway = new OutboundReverseGateway({gatewayId: "o2", connectionId: "detach_barrier_gateway", hub: {host: "127.0.0.1", port: hub.port, token: "gateway-token"}, tls, authorizeTarget: () => true, createTargetConnection: () => {
+    const target = new HeldWriteTarget()
+    targets.push(target)
+    return /** @type {import('node:net').Socket} */ (/** @type {unknown} */ (target))
+  }})
+  t.after(() => gateway.close())
+  t.after(() => { for (const target of targets) target.release() })
+  await gateway.start()
+
+  /** @type {import('node:tls').TLSSocket | undefined} */
+  let physical
+  const spoolDirectory = join(root, "client-spool")
+  const client = new ReverseGatewayConnection({gatewayId: "o2", connectionId: "detach_barrier_client", hub: {host: "127.0.0.1", port: hub.port, token: "client-token"}, tls, spoolDirectory, createPhysicalSocket: (options) => {
+    physical = connectTls({host: options.host, port: options.port, ...options.tls})
+    return physical
+  }})
+  t.after(() => client.close())
+  await client.connect()
+  const socket = client.createConnection({identifier: "detach_barrier_stream", host: "held-target", port: 2375})
+  await onceEvent(socket, "connect")
+  assert(physical !== undefined)
+
+  const payload = Buffer.from("queued-after-detach-".repeat(64))
+  const encoded = await encodeV3Data(payload, ZSTD_CODEC, new AdaptiveCompressor())
+  assert.notEqual(encoded.flags, 0, "fixture DATA must cross the asynchronous decompression boundary")
+  const originalDecompress = zlib.zstdDecompress
+  assert.equal(typeof originalDecompress, "function")
+  /** @type {(() => void) | undefined} */
+  let releaseDecompression
+  /** @type {() => void} */
+  let markDecompressionStarted = () => {}
+  const decompressionStarted = new Promise((resolve) => { markDecompressionStarted = () => resolve(undefined) })
+  /** @type {() => void} */
+  let markDecompressionFinished = () => {}
+  const decompressionFinished = new Promise((resolve) => { markDecompressionFinished = () => resolve(undefined) })
+  zlib.zstdDecompress = /** @type {typeof zlib.zstdDecompress} */ ((input, options, callback) => {
+    markDecompressionStarted()
+    releaseDecompression = () => originalDecompress(input, options, (error, result) => {
+      callback(error, result)
+      markDecompressionFinished()
+    })
+  })
+  t.after(() => {
+    releaseDecompression?.()
+    zlib.zstdDecompress = originalDecompress
+  })
+
+  physical.emit("data", encodeV3Frame({type: V3FrameType.DATA, streamId: socket.streamId, sequence: payload.length, flags: encoded.flags}, encoded.payload))
+  await withTimeout(decompressionStarted, "queued reverse DATA decompression")
+  let detached = false
+  const detaching = client.detach().then(() => { detached = true })
+  await new Promise((resolve) => setImmediate(resolve))
+  const resolvedBeforeQueue = detached
+  releaseDecompression?.()
+  releaseDecompression = undefined
+  await withTimeout(decompressionFinished, "released reverse DATA decompression")
+  await withTimeout(detaching, "reverse client detach barrier")
+  await new Promise((resolve) => setImmediate(resolve))
+
+  assert.equal(resolvedBeforeQueue, false, "detach must not resolve while an old-generation frame handler is queued")
+  assert.equal(socket.readableLength, 0, "a queued old-generation frame must not deliver after detach starts")
+  const recovery = JSON.parse(await readFile(join(spoolDirectory, "detach_barrier_stream", "recovery.json"), "utf8"))
+  assert.equal(recovery.receiveSequence, 0)
+  assert.equal(recovery.receiveAck, 0)
+})
+
+test("replacement gateway submits an empty resume manifest and resets omitted streams once", async (t) => {
+  const root = await mkdtemp(join(tmpdir(), "socketduct-reverse-gateway-replacement-"))
+  t.after(() => rm(root, {recursive: true, force: true}))
+  const certificates = generateTestCertificates(root)
+  let targetAccepts = 0
+  const targetSockets = new Set()
+  const target = createServer((socket) => { targetAccepts += 1; targetSockets.add(socket); socket.once("close", () => targetSockets.delete(socket)) })
+  const targetPort = await listen(target)
+  t.after(() => { for (const socket of targetSockets) socket.destroy(); return new Promise((resolve) => target.close(() => resolve(undefined))) })
+  const tokens = new StaticTokenStore([{username: "home", token: "gateway-token"}, {username: "app", token: "client-token"}])
+  const hub = await startReverseGatewayHub({tokens, tls: {certFile: certificates.serverCertFile, keyFile: certificates.serverKeyFile}, authorizeGateway: () => true, authorizeClientGateway: () => true, authorizeOpen: () => true, resumeWindowMs: 2_000})
+  t.after(() => hub.close())
+  const tls = {caFile: certificates.caCertFile, servername: "localhost"}
+  const gatewayOptions = {gatewayId: "o2", connectionId: "replacement_gateway_01", hub: {host: "127.0.0.1", port: hub.port, token: "gateway-token"}, tls, authorizeTarget: () => true}
+  const originalGateway = new OutboundReverseGateway({...gatewayOptions, reconnectDelayMs: 10_000, maxReconnectDelayMs: 10_000})
+  await originalGateway.start()
+  const client = new ReverseGatewayConnection({gatewayId: "o2", connectionId: "replacement_gateway_client", hub: {host: "127.0.0.1", port: hub.port, token: "client-token"}, tls, reconnectDelayMs: 10, maxReconnectDelayMs: 20})
+  t.after(() => client.close())
+  await client.connect()
+  const socket = client.createConnection({identifier: "replacement_stream_01", host: "127.0.0.1", port: targetPort})
+  let resets = 0
+  const reset = new Promise((resolve) => socket.on("error", (error) => { resets += 1; resolve(error) }))
+  await onceEvent(socket, "connect")
+  assert.equal(targetAccepts, 1)
+  const generation = originalGateway.status().connectionGeneration
+  await hub.closeActiveGateway("o2")
+  await originalGateway.close()
+
+  let replacementTargetCreates = 0
+  const replacement = new OutboundReverseGateway({...gatewayOptions, initialGeneration: generation, createTargetConnection: () => {
+    replacementTargetCreates += 1
+    throw new Error("replacement gateway must not recreate a retained target")
+  }})
+  t.after(() => replacement.close())
+  await replacement.start()
+  const error = /** @type {Error & {code?: string}} */ (await withTimeout(reset, "omitted replacement stream reset"))
+  assert.equal(error.code, "RESUME_SESSION_LOST")
+  await waitUntil(() => hub.status().virtualStreams === 0, "omitted bridge cleanup")
+  await new Promise((resolve) => setImmediate(resolve))
+  assert.equal(resets, 1)
+  assert.equal(replacementTargetCreates, 0)
+  assert.equal(replacement.status().targetOpens, 0)
+  assert.equal(targetAccepts, 1)
+})
+
 class DataThenErrorTarget extends Duplex {
   /** @param {Buffer} payload - Bytes observed before the terminal target error. */
   constructor(payload) {
@@ -1111,6 +1559,17 @@ test("reverse hub rejects bad credentials, unknown gateways, duplicate owners, a
   await replacement.start()
   await waitUntil(() => primary.status().physicalState === "closed", "newer generation fences prior owner")
   assert.equal(hub.status().healthyGateways, 1)
+
+  const primaryClient = new ReverseGatewayConnection({gatewayId: "o2", connectionId: "stable_client_owner", initialGeneration: 100, hub: {host: "127.0.0.1", port: hub.port, token: "client-valid-token"}, tls})
+  t.after(() => primaryClient.close())
+  await primaryClient.connect()
+  const equalClient = new ReverseGatewayConnection({gatewayId: "o2", connectionId: "stable_client_owner", initialGeneration: 100, hub: {host: "127.0.0.1", port: hub.port, token: "client-valid-token"}, tls, continuousReconnect: false})
+  t.after(() => equalClient.close())
+  await assert.rejects(withTimeout(equalClient.connect(), "equal client generation"), (error) => /** @type {{code?: string}} */ (error).code === "STALE_GENERATION")
+  const newerClient = new ReverseGatewayConnection({gatewayId: "o2", connectionId: "stable_client_owner", initialGeneration: 200, hub: {host: "127.0.0.1", port: hub.port, token: "client-valid-token"}, tls})
+  t.after(() => newerClient.close())
+  await newerClient.connect()
+  await waitUntil(() => primaryClient.status().physicalState === "closed", "newer client generation fences prior claimant")
 })
 
 test("hub and gateway both enforce target policy without opening a proxy", async (t) => {
@@ -1748,6 +2207,143 @@ test("an expired reverse resume window fails the logical stream once and release
   assert.equal(targets[0]?.destroyed, true)
 })
 
+test("one gateway disconnect incident preserves physical cause across fan-out and terminal resume errors", async (t) => {
+  const root = await mkdtemp(join(tmpdir(), "socketduct-reverse-attribution-"))
+  t.after(() => rm(root, {recursive: true, force: true}))
+  const certificates = generateTestCertificates(root)
+  const tokens = new StaticTokenStore([{username: "home", token: "gateway-attribution-token"}, {username: "app", token: "client-attribution-token"}])
+  /** @type {Array<Record<string, unknown> & {event: string}>} */
+  const hubEvents = []
+  /** @type {Array<Record<string, unknown> & {event: string}>} */
+  const gatewayEvents = []
+  /** @type {Array<Record<string, unknown> & {event: string}>} */
+  const clientEvents = []
+  const hub = await startReverseGatewayHub({
+    tokens,
+    tls: {certFile: certificates.serverCertFile, keyFile: certificates.serverKeyFile},
+    authorizeGateway: () => true,
+    authorizeClientGateway: () => true,
+    authorizeOpen: () => true,
+    resumeWindowMs: 40,
+    logger: {log: (/** @type {string} */ event, /** @type {Record<string, unknown>} */ fields = {}) => { hubEvents.push({event, ...fields}) }},
+    connectionFingerprint: {key: Buffer.alloc(32, 0x5a), keyEpoch: "test-epoch"}
+  })
+  t.after(() => hub.close())
+  const tls = {caFile: certificates.caCertFile, servername: "localhost"}
+  const targets = []
+  const gateway = new OutboundReverseGateway({
+    gatewayId: "o2",
+    connectionId: "low_entropy_gateway_connection",
+    hub: {host: "127.0.0.1", port: hub.port, token: "gateway-attribution-token"},
+    tls,
+    authorizeTarget: () => true,
+    reconnectDelayMs: 150,
+    maxReconnectDelayMs: 150,
+    logger: {log: (/** @type {string} */ event, /** @type {Record<string, unknown>} */ fields = {}) => { gatewayEvents.push({event, ...fields}) }},
+    createTargetConnection: () => {
+      const target = new InterruptibleTarget()
+      targets.push(target)
+      return /** @type {import('node:net').Socket} */ (/** @type {unknown} */ (target))
+    }
+  })
+  t.after(() => gateway.close())
+  await gateway.start()
+  const client = new ReverseGatewayConnection({
+    gatewayId: "o2",
+    connectionId: "low_entropy_client_connection",
+    hub: {host: "127.0.0.1", port: hub.port, token: "client-attribution-token"},
+    tls,
+    reconnectDelayMs: 5,
+    maxReconnectDelayMs: 10,
+    logger: {log: (/** @type {string} */ event, /** @type {Record<string, unknown>} */ fields = {}) => { clientEvents.push({event, ...fields}) }}
+  })
+  t.after(() => client.close())
+  await client.connect()
+  const sockets = [client.createConnection({host: "target", port: 2375}), client.createConnection({host: "target", port: 2375})]
+  await Promise.all(sockets.map((socket) => onceEvent(socket, "connect")))
+  const terminalErrors = sockets.map((socket) => nextError(socket))
+
+  await hub.closeActiveGateway("o2")
+  const errors = /** @type {Array<Error & Record<string, unknown>>} */ (await Promise.all(terminalErrors))
+  assert.deepEqual(errors.map((error) => error.code), ["RESUME_WINDOW_EXPIRED", "RESUME_WINDOW_EXPIRED"])
+  assert.equal(new Set(errors.map((error) => error.disconnectIncidentId)).size, 1)
+  for (const error of errors) {
+    assert.match(/** @type {string} */ (error.disconnectIncidentId), /^[A-Za-z0-9_-]{22}$/)
+    assert.equal(error.disconnectReason, "unknown_transport_close")
+    assert.equal(error.incidentOrigin, "reverse_hub")
+    assert.equal(error.physicalRole, "gateway")
+    assert.equal(typeof error.clientConnectionGeneration, "number")
+    assert.equal(typeof error.gatewayConnectionGeneration, "number")
+    assert.equal(error.fanOutCount, 2)
+  }
+  const detached = hubEvents.find(({event, physicalRole}) => event === "reverse_hub.physical_detached" && physicalRole === "gateway")
+  assert(detached !== undefined)
+  assert.equal(detached.disconnectIncidentId, /** @type {Error & Record<string, unknown>} */ (errors[0]).disconnectIncidentId)
+  assert.match(/** @type {string} */ (detached.connectionFingerprint), /^[a-f0-9]{32}$/)
+  assert.equal(detached.fingerprintKeyEpoch, "test-epoch")
+  assert(hubEvents.some(({event, fanOutCount}) => event === "reverse_hub.streams_interrupted" && fanOutCount === 2))
+  assert(hubEvents.some(({event, terminalStreamCount}) => event === "reverse_hub.streams_reset" && terminalStreamCount === 2))
+  assert(clientEvents.some(({event}) => event === "reverse_client.physical_detached"))
+  assert(gatewayEvents.some(({event}) => event === "reverse_gateway.physical_detached"))
+  assert.equal(targets.length, 2)
+  assert.doesNotMatch(JSON.stringify([hubEvents, gatewayEvents, clientEvents]), /low_entropy_(?:gateway|client)_connection|attribution-token/)
+})
+
+test("peer GOAWAY classification and hub attribution reach terminal non-resumable RESET", async (t) => {
+  const root = await mkdtemp(join(tmpdir(), "socketduct-reverse-terminal-attribution-"))
+  t.after(() => rm(root, {recursive: true, force: true}))
+  const certificates = generateTestCertificates(root)
+  const tokens = new StaticTokenStore([{username: "home", token: "gateway-token"}, {username: "app", token: "client-token"}])
+  /** @type {Array<Record<string, unknown> & {event: string}>} */
+  const hubEvents = []
+  const hub = await startReverseGatewayHub({
+    tokens,
+    tls: {certFile: certificates.serverCertFile, keyFile: certificates.serverKeyFile},
+    authorizeGateway: () => true,
+    authorizeClientGateway: () => true,
+    authorizeOpen: () => true,
+    logger: {log: (/** @type {string} */ event, /** @type {Record<string, unknown>} */ fields = {}) => { hubEvents.push({event, ...fields}) }}
+  })
+  t.after(() => hub.close())
+  const tls = {caFile: certificates.caCertFile, servername: "localhost"}
+  const gateway = new OutboundReverseGateway({
+    gatewayId: "o2",
+    connectionId: "terminal_attribution_gateway",
+    hub: {host: "127.0.0.1", port: hub.port, token: "gateway-token"},
+    tls,
+    authorizeTarget: () => true,
+    createTargetConnection: () => /** @type {import('node:net').Socket} */ (/** @type {unknown} */ (new InterruptibleTarget())),
+    createPhysicalSocket: (options) => {
+      const physical = connectTls({host: options.host, port: options.port, ...options.tls})
+      const originalWrite = physical.write.bind(physical)
+      physical.write = /** @type {typeof physical.write} */ ((chunk, ...args) => {
+        if (!Buffer.isBuffer(chunk) || chunk[5] !== V3FrameType.GOAWAY) return Reflect.apply(originalWrite, physical, [chunk, ...args])
+        const safeGoaway = encodeV3Frame({type: V3FrameType.GOAWAY, streamId: 0, sequence: 0}, encodeV3Control({code: "GATEWAY_MAINTENANCE", disconnectIncidentId: "BBBBBBBBBBBBBBBBBBBBBB"}))
+        return Reflect.apply(originalWrite, physical, [safeGoaway, ...args])
+      })
+      return physical
+    }
+  })
+  t.after(() => gateway.close())
+  await gateway.start()
+  const client = new ReverseGatewayConnection({gatewayId: "o2", connectionId: "terminal_attribution_client", hub: {host: "127.0.0.1", port: hub.port, token: "client-token"}, tls, continuousReconnect: false})
+  t.after(() => client.close())
+  await client.connect()
+  const socket = client.createConnection({host: "target", port: 2375})
+  await onceEvent(socket, "connect")
+  const terminal = nextError(socket)
+  await gateway.close()
+  const error = /** @type {Error & Record<string, unknown>} */ (await withTimeout(terminal, "terminal peer GOAWAY reset"))
+  assert.equal(error.code, "GATEWAY_MAINTENANCE")
+  assert.equal(error.disconnectReason, "peer_goaway")
+  assert.equal(error.incidentOrigin, "reverse_hub")
+  assert.equal(error.physicalRole, "gateway")
+  assert.equal(error.fanOutCount, 1)
+  assert.equal(error.terminalStreamCount, 1)
+  assert.notEqual(error.disconnectIncidentId, "BBBBBBBBBBBBBBBBBBBBBB")
+  assert(hubEvents.some(({event, terminalStreamCount}) => event === "reverse_hub.streams_reset" && terminalStreamCount === 1), JSON.stringify(hubEvents))
+})
+
 test("reverse mode enforces verified TLS and stream resource bounds", async (t) => {
   const root = await mkdtemp(join(tmpdir(), "socketduct-reverse-limits-"))
   const wrongRoot = await mkdtemp(join(tmpdir(), "socketduct-reverse-wrong-ca-"))

package/test/integration/reverse-route-class.test.js

@@ -364,7 +364,12 @@ test("a backpressured gateway does not block DATA or a new OPEN for another rout
     t.after(() => socket.destroy())
     return nextError(socket)
   })
-  assert.equal((await withTimeout(streamAError, "gateway A output limit")).code, "OUTPUT_BACKPRESSURE_LIMIT")
+  const outputError = /** @type {Error & Record<string, unknown>} */ (await withTimeout(streamAError, "gateway A output limit"))
+  assert.equal(outputError.code, "OUTPUT_BACKPRESSURE_LIMIT")
+  assert.equal(outputError.disconnectReason, "output_backpressure_limit")
+  assert.equal(outputError.incidentOrigin, "reverse_hub")
+  assert.equal(outputError.physicalRole, "gateway")
+  assert.match(/** @type {string} */ (outputError.disconnectIncidentId), /^[A-Za-z0-9_-]{22}$/)
   await withTimeout(Promise.all(queuedAErrors), "queued gateway A stream cleanup")
   assert.equal(await exchange(streamB, "after-limit"), "b:after-limit")
   assert.equal(hub.status().healthyGateways, 1)