socketduct 0.1.18 → 0.1.20

package/src/reverse/hub.js

@@ -2,6 +2,7 @@
 import {readFileSync} from "node:fs"
 import {createServer as createTlsServer} from "node:tls"
 
+import {errorDiagnostics, logDiagnostic} from "../logging/structured-logger.js"
 import {availableCodecs, negotiateCodec, zstdAvailable} from "../protocol/compression.js"
 import {decodeV3Data, encodeV3Data, AdaptiveCompressor} from "../protocol/v3-compression.js"
 import {decodeV3Control, encodeV3Control, encodeV3Frame, V3FrameDecoder, V3FrameType} from "../protocol/v3-frame.js"
@@ -22,6 +23,15 @@ import {
   STABLE_IDENTIFIER,
   writeSocket
 } from "./common.js"
+import {
+  connectionFingerprint,
+  createDisconnectAttribution,
+  disconnectReasonForCode,
+  parseDisconnectAttribution,
+  safeSocketCode,
+  sanitizePeerControlDetail,
+  validateFingerprintKeyEpoch
+} from "./disconnect-attribution.js"
 
 const DEFAULT_OUTPUT_BUFFER_BYTES = 32 * 1024 * 1024
 const DEFAULT_RESUME_MS = 60_000
@@ -52,6 +62,8 @@ const MANIFEST_PAGE_STREAMS = 64
  * @property {number} [maxOutputBytesPerConnection] - Per-physical queued hub output bound.
  * @property {number} [resumeWindowMs] - Detached reverse-stream retention window.
  * @property {import('../protocol/compression.js').CompressionLimits} [compressionLimits] - zstd safety bounds.
+ * @property {import('../logging/structured-logger.js').DiagnosticLogger} [logger] - Optional privacy-safe structured logger.
+ * @property {{key: Buffer | Uint8Array, keyEpoch: string}} [connectionFingerprint] - Optional hub-only correlation key and public epoch.
  * @typedef {object} HubPhysical
  * @property {import('node:tls').TLSSocket} socket
  * @property {Principal | undefined} principal
@@ -78,6 +90,10 @@ const MANIFEST_PAGE_STREAMS = 64
  * @property {HubOutputItem[]} outputQueue
  * @property {number} outputBytes
  * @property {boolean} outputWriting
+ * @property {import('./disconnect-attribution.js').DisconnectReason | undefined} firstDisconnectReason
+ * @property {import('./disconnect-attribution.js').DisconnectAttribution | undefined} disconnectAttribution
+ * @property {string | undefined} connectionFingerprint
+ * @property {string | undefined} safeSocketCode
  * @typedef {object} HubOutputItem
  * @property {Buffer} bytes
  * @property {(delivered: boolean) => void} resolve
@@ -119,6 +135,7 @@ const MANIFEST_PAGE_STREAMS = 64
  * @property {{generation: number, clientToGatewaySequence: number, clientToGatewayAck: number, gatewayToClientSequence: number, gatewayToClientAck: number, localEnded: boolean, remoteEnded: boolean} | undefined} clientResume
  * @property {{generation: number, clientToGatewaySequence: number, clientToGatewayAck: number, gatewayToClientSequence: number, gatewayToClientAck: number, localEnded: boolean, remoteEnded: boolean} | undefined} gatewayResume
  * @property {string | undefined} reconciledResume
+ * @property {import('./disconnect-attribution.js').DisconnectAttribution | undefined} disconnectAttribution
  */
 
 /** Starts the Server3 reverse hub. */
@@ -141,6 +158,8 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
   const tcpSockets = new Set()
   /** @type {Map<string, {socket: import('node:net').Socket, timer: ReturnType<typeof setTimeout>}>} */
   const pendingTls = new Map()
+  /** @type {Map<string, {fanOutCount: number, resumeSuccessCount: number, terminalStreamCount: number}>} */
+  const incidents = new Map()
   let bridges = 0
   let physicalAccepts = 0
   let gatewayRegistrations = 0
@@ -209,7 +228,11 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
       clearHandshake: undefined,
       outputQueue: [],
       outputBytes: 0,
-      outputWriting: false
+      outputWriting: false,
+      firstDisconnectReason: undefined,
+      disconnectAttribution: undefined,
+      connectionFingerprint: undefined,
+      safeSocketCode: undefined
     }
     physicals.add(physical)
     const decoder = new V3FrameDecoder({maxFrameBytes: Math.max(MAX_CONTROL_BYTES, maxFrameBytes + 4), maxBufferedBytes: Math.max(MAX_CONTROL_BYTES, maxFrameBytes + 4) * 2})
@@ -232,7 +255,11 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
         if (!socket.destroyed && !physical.closing) socket.resume()
       })
     })
-    socket.on("error", () => {})
+    socket.once("end", () => { physical.firstDisconnectReason ??= "remote_eof" })
+    socket.on("error", (error) => {
+      physical.firstDisconnectReason ??= "remote_socket_error"
+      physical.safeSocketCode ??= safeSocketCode(error)
+    })
     socket.once("close", () => {
       clearTimeout(handshakeTimer)
       void queue.finally(() => detachPhysical(physical, physical.detachCode ?? "TRANSPORT_CLOSED"))
@@ -257,6 +284,7 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
       physical.principal = principal
       physical.connectionId = auth.connectionId
       physical.generation = /** @type {number} */ (auth.generation)
+      if (options.connectionFingerprint !== undefined) physical.connectionFingerprint = connectionFingerprint(options.connectionFingerprint.key, physical.role, physical.connectionId)
       physical.codec = negotiateCodec(auth.compression, availableCodecs())
       if (auth.role === "client") authenticateClientOwner(physical, maxIdentities)
       physical.authenticated = true
@@ -277,6 +305,15 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
     }
     if (frame.type === V3FrameType.PONG) return
     if (frame.type === V3FrameType.GOAWAY) {
+      const detail = decodeV3Control(frame.payload)
+      const sanitized = sanitizePeerControlDetail(detail)
+      if (sanitized.reservedFieldsStripped > 0) logDiagnostic(options.logger, "reverse_hub.reserved_control_field_stripped", {
+        physicalRole: physical.role,
+        ...(physical.role === "client" ? {clientConnectionGeneration: physical.generation} : {gatewayConnectionGeneration: physical.generation}),
+        reservedFieldsStripped: sanitized.reservedFieldsStripped
+      })
+      physical.firstDisconnectReason ??= "peer_goaway"
+      if (typeof sanitized.detail.code === "string") physical.detachCode ??= sanitized.detail.code
       physical.retainOnDetach = false
       physical.socket.end()
       return
@@ -447,7 +484,7 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
         continue
       }
       if (bridge.identifier !== item.identifier) {
-        closeBridge(bridge, "RESUME_IDENTITY_MISMATCH", physical)
+        await closeBridge(bridge, "RESUME_IDENTITY_MISMATCH", physical)
         physical.resumeEntries.push({streamId: item.streamId, bridge: undefined, code: "RESUME_IDENTITY_MISMATCH"})
         continue
       }
@@ -464,7 +501,7 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
     }
     for (const bridge of [...physical.streams.values()]) {
       const streamId = physical.role === "client" ? bridge.clientStreamId : bridge.gatewayStreamId
-      if (!presented.has(streamId)) closeBridge(bridge, "RESUME_STREAM_OMITTED", physical)
+      if (!presented.has(streamId)) await closeBridge(bridge, "RESUME_STREAM_OMITTED", physical)
     }
     physical.resumeItems = []
     physical.resumeFinal = true
@@ -507,7 +544,7 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
           try {
             reconcileBridge(bridge)
           } catch {
-            closeBridge(bridge, "RESUME_STATE_MISMATCH")
+            await closeBridge(bridge, "RESUME_STATE_MISMATCH")
             results.push({streamId: entry.streamId, status: "reset", code: "RESUME_STATE_MISMATCH"})
             continue
           }
@@ -575,6 +612,18 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
     bridge.clientToGatewayCompressor = new AdaptiveCompressor()
     bridge.gatewayToClientCompressor = new AdaptiveCompressor()
     bridge.reconciledResume = key
+    if (bridge.disconnectAttribution !== undefined) {
+      const attribution = bridge.disconnectAttribution
+      const incident = incidentState(attribution)
+      incident.resumeSuccessCount += 1
+      logDiagnostic(options.logger, "reverse_hub.streams_resumed", {
+        ...currentIncidentAttribution(attribution),
+        gatewayId: bridge.gatewayOwner.gatewayId,
+        ...(bridge.client.routeClass === undefined ? {} : {routeClass: bridge.client.routeClass})
+      })
+      bridge.disconnectAttribution = undefined
+      releaseCompletedIncident(attribution)
+    }
   }
 
   /** @param {Record<string, unknown>} value - Resume manifest entry. */
@@ -668,7 +717,8 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
       gatewayToClientCompressor: new AdaptiveCompressor(),
       clientResume: undefined,
       gatewayResume: undefined,
-      reconciledResume: undefined
+      reconciledResume: undefined,
+      disconnectAttribution: undefined
     }
     clientOwner.streams.set(frame.streamId, bridge)
     gatewayOwner.streams.set(gatewayStreamId, bridge)
@@ -711,7 +761,7 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
     }
     if (bridge.state === "closing") {
       if (frame.type === V3FrameType.DATA || frame.type === V3FrameType.END) {
-        closeBridge(bridge, "STREAM_CLOSING")
+        await closeBridge(bridge, "STREAM_CLOSING")
         return
       }
       if (frame.type !== V3FrameType.CLOSE && frame.type !== V3FrameType.RESET) throw protocolError("STREAM_CLOSING")
@@ -742,10 +792,32 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
     if (frame.type === V3FrameType.CLOSE || frame.type === V3FrameType.RESET) {
       const expected = fromClient ? bridge.clientToGatewaySequence : bridge.gatewayToClientSequence
       if ((frame.type === V3FrameType.CLOSE && frame.sequence !== expected) || (frame.type === V3FrameType.RESET && frame.sequence !== 0)) throw protocolError("INVALID_STREAM_CLOSE")
-      const detail = frame.payload.length === 0 ? {} : decodeV3Control(frame.payload)
+      const peerDetail = frame.payload.length === 0 ? {} : decodeV3Control(frame.payload)
+      const sanitized = sanitizePeerControlDetail(peerDetail)
+      if (sanitized.reservedFieldsStripped > 0) logDiagnostic(options.logger, "reverse_hub.reserved_control_field_stripped", {
+        physicalRole: source.role,
+        ...(source.role === "client" ? {clientConnectionGeneration: source.generation} : {gatewayConnectionGeneration: source.generation}),
+        reservedFieldsStripped: sanitized.reservedFieldsStripped
+      })
+      let attribution
+      if (frame.type === V3FrameType.RESET && bridge.disconnectAttribution !== undefined) {
+        incidentState(bridge.disconnectAttribution).terminalStreamCount += 1
+        attribution = currentIncidentAttribution(bridge.disconnectAttribution)
+      }
+      const detail = attribution === undefined ? sanitized.detail : {...sanitized.detail, ...attribution}
       forwardControl(destination, frame.type, destinationStreamId, frame.sequence, detail)
-      if (frame.type === V3FrameType.RESET || fromClient) removeBridge(bridge)
-      else bridge.state = "closing"
+      if (frame.type === V3FrameType.RESET || fromClient) {
+        removeBridge(bridge)
+        if (attribution !== undefined && bridge.disconnectAttribution !== undefined) {
+          logDiagnostic(options.logger, "reverse_hub.streams_reset", {
+            ...attribution,
+            gatewayId: bridge.gatewayOwner.gatewayId,
+            ...(bridge.client.routeClass === undefined ? {} : {routeClass: bridge.client.routeClass}),
+            ...errorDiagnostics(sanitized.detail, "STREAM_RESET")
+          })
+          releaseCompletedIncident(bridge.disconnectAttribution)
+        }
+      } else bridge.state = "closing"
       return
     }
     throw protocolError("UNSUPPORTED_FRAME")
@@ -768,44 +840,76 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
     forwardOutput(destination, encodeV3Frame({type, streamId, sequence}, encodeV3Control(value)))
   }
 
-  /** @param {Bridge} bridge - Stream. @param {string} code - Reset code. @param {HubPhysical} [excluded] - Already-dead peer. */
+  /** @param {Bridge} bridge - Stream. @param {string} code - Reset code. @param {HubPhysical} [excluded] - Already-dead peer. @returns {Promise<void>} Terminal control delivery. */
   function closeBridge(bridge, code, excluded) {
-    if (bridge.client !== excluded && !bridge.client.socket.destroyed) void sendControl(bridge.client, V3FrameType.RESET, bridge.clientStreamId, 0, {code})
-    if (bridge.gateway !== excluded && !bridge.gateway.socket.destroyed) void sendControl(bridge.gateway, V3FrameType.RESET, bridge.gatewayStreamId, 0, {code})
+    const owned = bridgeStillOwned(bridge)
+    let attribution
+    if (owned && bridge.disconnectAttribution !== undefined) {
+      const incident = incidentState(bridge.disconnectAttribution)
+      incident.terminalStreamCount += 1
+      attribution = currentIncidentAttribution(bridge.disconnectAttribution)
+    }
+    const detail = {code, ...(attribution ?? {})}
+    const deliveries = []
+    if (bridge.client !== excluded && !bridge.client.socket.destroyed) deliveries.push(sendControl(bridge.client, V3FrameType.RESET, bridge.clientStreamId, 0, detail))
+    if (bridge.gateway !== excluded && !bridge.gateway.socket.destroyed) deliveries.push(sendControl(bridge.gateway, V3FrameType.RESET, bridge.gatewayStreamId, 0, detail))
     removeBridge(bridge)
+    if (attribution !== undefined) logDiagnostic(options.logger, "reverse_hub.streams_reset", {
+      ...attribution,
+      gatewayId: bridge.gatewayOwner.gatewayId,
+      ...(bridge.client.routeClass === undefined ? {} : {routeClass: bridge.client.routeClass}),
+      ...errorDiagnostics({code})
+    })
+    if (bridge.disconnectAttribution !== undefined) releaseCompletedIncident(bridge.disconnectAttribution)
+    return Promise.all(deliveries).then(() => undefined)
   }
 
-  /** @param {HubPhysical} physical - Connection. @param {string} code - Reason. @param {boolean} [retainStreams] - Preserve logical streams for paired resume. */
-  function terminate(physical, code, retainStreams = false) {
+  /** @param {HubPhysical} physical - Connection. @param {string} code - Reason. @param {boolean} [retainStreams] - Preserve logical streams for paired resume. @param {import('./disconnect-attribution.js').DisconnectAttribution} [attribution] - Propagated first-cause incident. */
+  function terminate(physical, code, retainStreams = false, attribution = undefined) {
     if (physical.closing) return
     physical.closing = true
     physical.retainOnDetach = retainStreams
     physical.detachCode = code
+    const disconnectAttribution = ensurePhysicalAttribution(physical, disconnectReasonForCode(code), attribution)
     const forceTimer = setTimeout(() => physical.socket.destroy(), 1_000)
     forceTimer.unref?.()
-    void sendControl(physical, V3FrameType.GOAWAY, 0, 0, {code}, true).finally(() => {
+    void sendControl(physical, V3FrameType.GOAWAY, 0, 0, {code, ...(disconnectAttribution === undefined ? {} : currentIncidentAttribution(disconnectAttribution))}, true).finally(() => {
       clearTimeout(forceTimer)
       physical.socket.destroy()
     })
   }
 
   /** @param {HubPhysical} physical - Closed connection. @param {string} code - Stream reset. */
-  function detachPhysical(physical, code) {
+  async function detachPhysical(physical, code) {
     if (!physicals.delete(physical)) return
     physical.closing = true
+    const attribution = ensurePhysicalAttribution(physical, physical.firstDisconnectReason ?? disconnectReasonForCode(code))
+    if (attribution !== undefined) logDiagnostic(options.logger, "reverse_hub.physical_detached", {
+      ...currentIncidentAttribution(attribution),
+      ...(physical.connectionFingerprint === undefined || attribution.physicalRole !== physical.role ? {} : {connectionFingerprint: physical.connectionFingerprint, fingerprintKeyEpoch: options.connectionFingerprint?.keyEpoch}),
+      ...(physical.safeSocketCode === undefined ? {} : {safeSocketCode: physical.safeSocketCode})
+    })
     clearResumeTimeout(physical)
     discardQueuedOutput(physical)
     if (physical.gatewayId !== undefined) {
       const owner = gateways.get(physical.gatewayId)
       if (physical.role === "gateway" && owner?.physical === physical) {
         owner.physical = undefined
+        if (attribution !== undefined) for (const bridge of owner.streams.values()) assignBridgeAttribution(bridge, attribution)
         if (!physical.retainOnDetach || closing) {
-          for (const bridge of [...owner.streams.values()]) closeBridge(bridge, code, physical)
+          for (const bridge of [...owner.streams.values()]) await closeBridge(bridge, code, physical)
           gateways.delete(owner.gatewayId)
         } else {
+          if (attribution !== undefined) {
+            logDiagnostic(options.logger, "reverse_hub.streams_interrupted", {...currentIncidentAttribution(attribution), gatewayId: owner.gatewayId})
+          }
           scheduleOwnerExpiry(owner, "gateway")
           const clientsToReconnect = new Set([...owner.streams.values()].map((bridge) => bridge.clientOwner.physical).filter((peer) => peer !== undefined))
-          for (const client of clientsToReconnect) terminate(/** @type {HubPhysical} */ (client), "GATEWAY_DISCONNECTED", true)
+          for (const client of clientsToReconnect) {
+            const peer = /** @type {HubPhysical} */ (client)
+            const propagated = attribution === undefined ? undefined : createDisconnectAttribution({...currentIncidentAttribution(attribution), clientConnectionGeneration: peer.generation})
+            terminate(peer, "GATEWAY_DISCONNECTED", true, propagated)
+          }
         }
       }
     }
@@ -813,37 +917,109 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
       const owner = clients.get(physical.connectionId)
       if (owner?.physical === physical) {
         owner.physical = undefined
+        if (attribution !== undefined) for (const bridge of owner.streams.values()) assignBridgeAttribution(bridge, attribution)
         if (!physical.retainOnDetach || closing) {
-          for (const bridge of [...owner.streams.values()]) closeBridge(bridge, code, physical)
+          for (const bridge of [...owner.streams.values()]) await closeBridge(bridge, code, physical)
           clients.delete(owner.connectionId)
         } else {
+          if (attribution !== undefined) {
+            logDiagnostic(options.logger, "reverse_hub.streams_interrupted", {
+              ...currentIncidentAttribution(attribution),
+              ...(physical.gatewayId === undefined ? {} : {gatewayId: physical.gatewayId}),
+              ...(physical.routeClass === undefined ? {} : {routeClass: physical.routeClass})
+            })
+          }
           scheduleOwnerExpiry(owner, "client")
           const gatewaysToReconnect = new Set([...owner.streams.values()].map((bridge) => bridge.gatewayOwner.physical).filter((peer) => peer !== undefined))
-          for (const gateway of gatewaysToReconnect) terminate(/** @type {HubPhysical} */ (gateway), "CLIENT_DISCONNECTED", true)
+          for (const gateway of gatewaysToReconnect) {
+            const peer = /** @type {HubPhysical} */ (gateway)
+            const propagated = attribution === undefined ? undefined : createDisconnectAttribution({...currentIncidentAttribution(attribution), gatewayConnectionGeneration: peer.generation})
+            terminate(peer, "CLIENT_DISCONNECTED", true, propagated)
+          }
         }
       }
     }
+    if (attribution !== undefined && (!physical.retainOnDetach || incidentState(attribution).fanOutCount === 0)) incidents.delete(attribution.disconnectIncidentId)
+  }
+
+  /** @param {HubPhysical} physical - Connection. @param {import('./disconnect-attribution.js').DisconnectReason} reason - Physical cause. @param {import('./disconnect-attribution.js').DisconnectAttribution | undefined} [propagated] - Existing incident. */
+  function ensurePhysicalAttribution(physical, reason, propagated = undefined) {
+    if (physical.disconnectAttribution !== undefined) return physical.disconnectAttribution
+    if (physical.role === undefined) return undefined
+    const source = propagated === undefined ? undefined : parseDisconnectAttribution(propagated)
+    physical.disconnectAttribution = createDisconnectAttribution({
+      ...(source ?? {}),
+      disconnectReason: source?.disconnectReason ?? reason,
+      incidentOrigin: source?.incidentOrigin ?? "reverse_hub",
+      physicalRole: source?.physicalRole ?? physical.role,
+      ...(physical.role === "client" ? {clientConnectionGeneration: physical.generation} : {gatewayConnectionGeneration: physical.generation}),
+      fanOutCount: source?.fanOutCount ?? physical.streams.size,
+      resumeSuccessCount: source?.resumeSuccessCount ?? 0,
+      terminalStreamCount: source?.terminalStreamCount ?? 0
+    })
+    incidentState(physical.disconnectAttribution)
+    return physical.disconnectAttribution
+  }
+
+  /** Preserves the first incident while completing both endpoint generations. */
+  function assignBridgeAttribution(/** @type {Bridge} */ bridge, /** @type {import('./disconnect-attribution.js').DisconnectAttribution} */ attribution) {
+    bridge.disconnectAttribution ??= createDisconnectAttribution({
+      ...currentIncidentAttribution(attribution),
+      clientConnectionGeneration: bridge.client.generation,
+      gatewayConnectionGeneration: bridge.gateway.generation
+    })
+  }
+
+  /** @param {import('./disconnect-attribution.js').DisconnectAttribution} attribution - Incident. @returns {{fanOutCount: number, resumeSuccessCount: number, terminalStreamCount: number}} */
+  function incidentState(attribution) {
+    let state = incidents.get(attribution.disconnectIncidentId)
+    if (state === undefined) {
+      state = {fanOutCount: attribution.fanOutCount, resumeSuccessCount: attribution.resumeSuccessCount, terminalStreamCount: attribution.terminalStreamCount}
+      incidents.set(attribution.disconnectIncidentId, state)
+    } else {
+      state.fanOutCount = Math.max(state.fanOutCount, attribution.fanOutCount)
+      state.resumeSuccessCount = Math.max(state.resumeSuccessCount, attribution.resumeSuccessCount)
+      state.terminalStreamCount = Math.max(state.terminalStreamCount, attribution.terminalStreamCount)
+    }
+    return state
+  }
+
+  /** @param {import('./disconnect-attribution.js').DisconnectAttribution} attribution - Incident. */
+  function currentIncidentAttribution(attribution) {
+    const state = incidentState(attribution)
+    return createDisconnectAttribution({...attribution, ...state})
+  }
+
+  /** @param {import('./disconnect-attribution.js').DisconnectAttribution} attribution - Incident. */
+  function releaseCompletedIncident(attribution) {
+    const state = incidentState(attribution)
+    if (state.resumeSuccessCount + state.terminalStreamCount >= state.fanOutCount) incidents.delete(attribution.disconnectIncidentId)
   }
 
   /** @param {GatewayOwner | ClientOwner} owner - Detached identity. @param {'gateway' | 'client'} role - Owner role. */
   function scheduleOwnerExpiry(owner, role) {
     if (owner.expiryTimer !== undefined) clearTimeout(owner.expiryTimer)
     owner.expiryTimer = setTimeout(() => {
-      owner.expiryTimer = undefined
-      if (owner.physical !== undefined) return
-      for (const bridge of [...owner.streams.values()]) closeBridge(bridge, "RESUME_WINDOW_EXPIRED")
-      if (role === "gateway") {
-        const gatewayOwner = /** @type {GatewayOwner} */ (owner)
-        if (gateways.get(gatewayOwner.gatewayId) === gatewayOwner) gateways.delete(gatewayOwner.gatewayId)
-      } else {
-        const clientOwner = /** @type {ClientOwner} */ (owner)
-        if (clients.get(clientOwner.connectionId) === clientOwner) clients.delete(clientOwner.connectionId)
-      }
-      void tryCompleteResumes()
+      void expireOwner(owner, role)
     }, resumeWindowMs)
     owner.expiryTimer.unref?.()
   }
 
+  /** @param {GatewayOwner | ClientOwner} owner - Detached identity. @param {'gateway' | 'client'} role - Owner role. */
+  async function expireOwner(owner, role) {
+    owner.expiryTimer = undefined
+    if (owner.physical !== undefined) return
+    for (const bridge of [...owner.streams.values()]) await closeBridge(bridge, "RESUME_WINDOW_EXPIRED")
+    if (role === "gateway") {
+      const gatewayOwner = /** @type {GatewayOwner} */ (owner)
+      if (gateways.get(gatewayOwner.gatewayId) === gatewayOwner) gateways.delete(gatewayOwner.gatewayId)
+    } else {
+      const clientOwner = /** @type {ClientOwner} */ (owner)
+      if (clients.get(clientOwner.connectionId) === clientOwner) clients.delete(clientOwner.connectionId)
+    }
+    void tryCompleteResumes()
+  }
+
   /** @param {HubPhysical} physical - Destination. @param {Buffer} bytes - Encoded frame. @param {boolean} [allowClosing] - Allow a final GOAWAY after closing starts. @returns {Promise<boolean>} Delivery result. */
   function enqueueOutput(physical, bytes, allowClosing = false) {
     if (physical.socket.destroyed || (physical.closing && !allowClosing)) return Promise.resolve(false)
@@ -897,6 +1073,8 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
     physical.closing = true
     physical.retainOnDetach = false
     physical.detachCode ??= code
+    physical.firstDisconnectReason ??= disconnectReasonForCode(code)
+    ensurePhysicalAttribution(physical, physical.firstDisconnectReason)
     discardQueuedOutput(physical)
     if (!physical.socket.destroyed) physical.socket.destroy(protocolError(code))
   }
@@ -961,6 +1139,11 @@ function socketKey(socket) {
 /** @param {ReverseGatewayHubOptions} options - Options. */
 function validateOptions(options) {
   if (typeof options !== "object" || options === null || options.tokens === undefined || typeof options.tokens.authenticate !== "function") throw new TypeError("Reverse gateway hub requires a token authenticator")
+  if (options.connectionFingerprint !== undefined) {
+    const {key, keyEpoch} = options.connectionFingerprint
+    if (!(key instanceof Uint8Array) || key.byteLength < 32) throw new TypeError("connection fingerprint key must contain at least 32 bytes")
+    validateFingerprintKeyEpoch(keyEpoch)
+  }
   if (typeof options.authorizeGateway !== "function" || typeof options.authorizeClientGateway !== "function" || typeof options.authorizeOpen !== "function") throw new TypeError("Reverse gateway hub requires gateway, client selection, and open authorization policies")
   if (typeof options.tls !== "object" || options.tls === null || typeof options.tls.certFile !== "string" || typeof options.tls.keyFile !== "string") throw new TypeError("Reverse gateway hub requires TLS certFile and keyFile")
   if (!zstdAvailable()) throw new Error("Reverse gateway hub requires Node.js zstd support")

package/src/reverse/runtime.js

@@ -3,8 +3,8 @@ import {StaticTokenStore} from "../auth/static-token-store.js"
 import {OutboundReverseGateway} from "./gateway.js"
 import {startReverseGatewayHub} from "./hub.js"
 
-/** @param {import('../config/environment.js').RelayEnvironmentConfig} config - Parsed environment. */
-export async function startReverseHubRuntime(config) {
+/** @param {import('../config/environment.js').RelayEnvironmentConfig} config - Parsed environment. @param {import('../logging/structured-logger.js').DiagnosticLogger} [logger] - Optional structured logger. */
+export async function startReverseHubRuntime(config, logger) {
   if (config.mode !== "reverse-hub") throw new Error("Reverse hub runtime requires SOCKETDUCT_MODE=reverse-hub")
   if (config.relayTls === undefined) throw new Error("Reverse hub mode requires SOCKETDUCT_TLS_CERT_FILE and SOCKETDUCT_TLS_KEY_FILE")
   if (config.authTokens.length === 0) throw new Error("SOCKETDUCT_AUTH_TOKENS must contain hub client and gateway credentials")
@@ -54,7 +54,9 @@ export async function startReverseHubRuntime(config) {
     resumeWindowMs: config.resumeWindowMs,
     handshakeTimeoutMs: config.handshakeTimeoutMs,
     maxInputBytesPerConnection: config.maxInputBytesPerConnection,
-    compressionLimits: compressionLimits(config)
+    compressionLimits: compressionLimits(config),
+    ...(logger === undefined ? {} : {logger}),
+    ...(config.connectionFingerprint === undefined ? {} : {connectionFingerprint: config.connectionFingerprint})
   })
   return {hub, close: hub.close}
 }

package/src/spool/spool-file.js

@@ -1,4 +1,4 @@
-import {readFileSync, readdirSync, rmSync, statSync} from "node:fs"
+import {readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync} from "node:fs"
 import {appendFile, mkdir, readFile, rename, rm, stat, writeFile} from "node:fs/promises"
 import {join, resolve} from "node:path"
 
@@ -33,6 +33,7 @@ import {join, resolve} from "node:path"
 
 const VALID_SESSION_ID = /^[a-zA-Z0-9_-]{8,}$/
 const MAX_RECOVERY_STATE_BYTES = 16 * 1024
+const MAX_SERIALIZED_CHUNK_OVERHEAD = 64
 
 export class SpoolQuotaError extends Error {
   /**
@@ -112,12 +113,7 @@ export class SessionSpool {
    */
   readRecoveryStateSync(sessionId, ownerId) {
     try {
-      const directory = this.#sessionDirectory(sessionId)
-      const metadataPath = join(directory, "metadata.json")
-      if (statSync(metadataPath).size > MAX_RECOVERY_STATE_BYTES) throw new Error("recovery ownership metadata is oversized")
-      const metadata = JSON.parse(readFileSync(metadataPath, "utf8"))
-      if (metadata?.sessionId !== sessionId || metadata?.ownerId !== ownerId) throw new Error("recovery ownership mismatch")
-      this.#assertMetadataAge(metadata, sessionId)
+      const directory = this.#readRecoveryOwnerSync(sessionId, ownerId)
       const path = join(directory, "recovery.json")
       if (statSync(path).size > MAX_RECOVERY_STATE_BYTES) throw new Error("recovery metadata is oversized")
       const state = JSON.parse(readFileSync(path, "utf8"))
@@ -129,6 +125,97 @@ export class SessionSpool {
     }
   }
 
+  /**
+   * Loads durable chunks synchronously so recovery can fail before network
+   * activity. Ownership, age, encoding, accounting, and configured byte bounds
+   * are all checked before any chunk is returned.
+   * @param {string} sessionId - Stable session identifier.
+   * @param {string} ownerId - Expected spool owner.
+   * @param {number} [acknowledgedSequence] - Omit chunks already covered by durable recovery state.
+   * @param {number} [committedSequence] - Highest chunk boundary committed by durable recovery state.
+   * @returns {SpoolChunk[]} Parsed durable chunks after the acknowledgement.
+   */
+  loadChunksSync(sessionId, ownerId, acknowledgedSequence = 0, committedSequence = undefined) {
+    try {
+      if (!Number.isSafeInteger(acknowledgedSequence) || acknowledgedSequence < 0) throw new Error("recovery acknowledgement is invalid")
+      if (committedSequence !== undefined && (!Number.isSafeInteger(committedSequence) || committedSequence < acknowledgedSequence)) throw new Error("recovery committed sequence is invalid")
+      const directory = this.#readRecoveryOwnerSync(sessionId, ownerId)
+      const chunksPath = join(directory, "chunks.jsonl")
+      if (this.#maxBytes !== undefined) {
+        const serializedLimit = Math.min(Number.MAX_SAFE_INTEGER, Math.max(MAX_RECOVERY_STATE_BYTES, (this.#maxBytes * MAX_SERIALIZED_CHUNK_OVERHEAD) + MAX_RECOVERY_STATE_BYTES))
+        if (statSync(chunksPath).size > serializedLimit) throw new Error("recovery chunk storage is oversized")
+      }
+      const raw = readFileSync(chunksPath, "utf8")
+      /** @type {SpoolChunk[]} */
+      const chunks = []
+      let totalBytes = 0
+      let previousSequence
+      if (raw !== "") {
+        const lines = raw.split("\n")
+        if (lines.at(-1) === "") lines.pop()
+        for (const [index, line] of lines.entries()) {
+          let serialized
+          try {
+            serialized = /** @type {Partial<SerializedChunk>} */ (JSON.parse(line))
+          } catch (error) {
+            const partialTail = index === lines.length - 1 && !raw.endsWith("\n") && committedSequence !== undefined && committedChunksAreCovered(chunks, acknowledgedSequence, committedSequence)
+            if (partialTail) break
+            throw error
+          }
+          if (!Number.isSafeInteger(serialized.sequence) || /** @type {number} */ (serialized.sequence) < 0 || typeof serialized.dataBase64 !== "string") throw new Error("recovery chunk is malformed")
+          const data = Buffer.from(serialized.dataBase64, "base64")
+          if (data.length === 0 || data.toString("base64") !== serialized.dataBase64) throw new Error("recovery chunk encoding is invalid")
+          if (data.length > /** @type {number} */ (serialized.sequence) || (previousSequence !== undefined && serialized.sequence !== previousSequence + data.length)) throw new Error("recovery chunks are not contiguous")
+          previousSequence = /** @type {number} */ (serialized.sequence)
+          totalBytes += data.length
+          if (!Number.isSafeInteger(totalBytes) || (this.#maxBytes !== undefined && totalBytes > this.#maxBytes)) throw new Error("recovery chunks exceed the configured byte limit")
+          chunks.push({sequence: /** @type {number} */ (serialized.sequence), data})
+        }
+      }
+      const usagePath = join(directory, "usage.json")
+      if (statSync(usagePath).size > MAX_RECOVERY_STATE_BYTES) throw new Error("recovery usage metadata is oversized")
+      const usage = JSON.parse(readFileSync(usagePath, "utf8"))
+      if (!Number.isSafeInteger(usage?.totalBytes) || usage.totalBytes < 0) throw new Error("recovery chunk accounting is malformed")
+      if (committedSequence === undefined) {
+        if (usage.totalBytes !== totalBytes) throw new Error("recovery chunk accounting mismatch")
+        this.#usedBytes.set(sessionId, totalBytes)
+        this.#usageLoaded.add(sessionId)
+        return chunks.filter((chunk) => chunk.sequence > acknowledgedSequence)
+      }
+      if (!committedChunksAreCovered(chunks, acknowledgedSequence, committedSequence)) throw new Error("recovery chunks do not cover the committed sequence")
+      const firstAfterAck = chunks.find((chunk) => chunk.sequence > acknowledgedSequence)
+      if (firstAfterAck !== undefined && firstAfterAck.sequence - firstAfterAck.data.length !== acknowledgedSequence) throw new Error("recovery chunks do not start at the acknowledgement")
+      if (firstAfterAck === undefined && chunks.length > 0 && chunks.at(-1)?.sequence !== acknowledgedSequence) throw new Error("recovery acknowledgement is not a chunk boundary")
+      const committed = chunks.filter((chunk) => chunk.sequence > acknowledgedSequence && chunk.sequence <= committedSequence)
+      const committedBytes = committed.reduce((sum, chunk) => sum + chunk.data.length, 0)
+      if (!Number.isSafeInteger(committedBytes) || (this.#maxBytes !== undefined && committedBytes > this.#maxBytes)) throw new Error("recovery chunks exceed the configured byte limit")
+      const repairedChunks = serializeChunks(committed)
+      if (raw !== repairedChunks) writeFileAtomicSync(chunksPath, repairedChunks, {mode: 0o600})
+      if (usage.totalBytes !== committedBytes) writeFileAtomicSync(usagePath, JSON.stringify({totalBytes: committedBytes}) + "\n", {mode: 0o600})
+      this.#usedBytes.set(sessionId, committedBytes)
+      this.#usageLoaded.add(sessionId)
+      return committed
+    } catch (error) {
+      if (error instanceof SpoolQuotaError) throw error
+      throw new Error(`Invalid Socketduct recovery chunks for ${sessionId}`, {cause: error})
+    }
+  }
+
+  /**
+   * @param {string} sessionId - Stable session identifier.
+   * @param {string} ownerId - Expected owner.
+   * @returns {string} Validated session directory.
+   */
+  #readRecoveryOwnerSync(sessionId, ownerId) {
+    const directory = this.#sessionDirectory(sessionId)
+    const metadataPath = join(directory, "metadata.json")
+    if (statSync(metadataPath).size > MAX_RECOVERY_STATE_BYTES) throw new Error("recovery ownership metadata is oversized")
+    const metadata = JSON.parse(readFileSync(metadataPath, "utf8"))
+    if (metadata?.sessionId !== sessionId || metadata?.ownerId !== ownerId) throw new Error("recovery ownership mismatch")
+    this.#assertMetadataAge(metadata, sessionId)
+    return directory
+  }
+
   /**
    * Removes only well-formed, expired session directories beneath the private spool root.
    * @returns {void}
@@ -436,6 +523,48 @@ async function writeFileAtomic(path, contents, options) {
   await rename(tempPath, path)
 }
 
+/**
+ * @param {string} path - Path to replace.
+ * @param {string} contents - Complete durable contents.
+ * @param {{mode: number}} options - File mode.
+ * @returns {void}
+ */
+function writeFileAtomicSync(path, contents, options) {
+  const tempPath = `${path}.${process.pid}.${Date.now()}.tmp`
+  writeFileSync(tempPath, contents, {mode: options.mode})
+  renameSync(tempPath, path)
+}
+
+/**
+ * @param {readonly SpoolChunk[]} chunks - Ordered chunks.
+ * @returns {string} JSONL contents.
+ */
+function serializeChunks(chunks) {
+  const lines = chunks.map((chunk) => JSON.stringify(/** @type {SerializedChunk} */ ({
+    sequence: chunk.sequence,
+    dataBase64: chunk.data.toString("base64")
+  })))
+  return lines.length === 0 ? "" : `${lines.join("\n")}\n`
+}
+
+/**
+ * @param {readonly SpoolChunk[]} chunks - Structurally validated physical chunks.
+ * @param {number} acknowledgedSequence - Durable ACK boundary.
+ * @param {number} committedSequence - Durable append boundary.
+ * @returns {boolean} Whether the committed interval is complete and contiguous.
+ */
+function committedChunksAreCovered(chunks, acknowledgedSequence, committedSequence) {
+  if (committedSequence === acknowledgedSequence) return true
+  let sequence = acknowledgedSequence
+  for (const chunk of chunks) {
+    if (chunk.sequence <= acknowledgedSequence) continue
+    if (chunk.sequence > committedSequence) break
+    if (chunk.sequence - chunk.data.length !== sequence) return false
+    sequence = chunk.sequence
+  }
+  return sequence === committedSequence
+}
+
 /**
  * @param {string} directory - directory value.
  * @returns {Promise<void>} - Result value.

package/tensorbuzz.yml

@@ -1,9 +1,10 @@
+dependency_roots:
+  - path: .
 before_install:
   - curl -fsSL https://deb.nodesource.com/setup_24.x | sudo -E bash -
-  - sudo apt-get install -y docker.io nodejs
+  - sudo apt-get install -y docker.io nodejs zstd=1.5.7+dfsg-3
 before_script:
   - node --version
-  - npm ci
 environment:
   DOCKER_HOST: "tcp://docker_server:2375"
 services:
@@ -33,4 +34,5 @@ script:
   - npm run typecheck
   - npm run lint
   - npm test
+  - npm run verify:zstd-logs
   - docker build --tag socketduct-ci .