socketduct 0.1.19 → 0.1.21

package/src/reverse/hub.js

@@ -2,6 +2,7 @@
 import {readFileSync} from "node:fs"
 import {createServer as createTlsServer} from "node:tls"
 
+import {errorDiagnostics, logDiagnostic} from "../logging/structured-logger.js"
 import {availableCodecs, negotiateCodec, zstdAvailable} from "../protocol/compression.js"
 import {decodeV3Data, encodeV3Data, AdaptiveCompressor} from "../protocol/v3-compression.js"
 import {decodeV3Control, encodeV3Control, encodeV3Frame, V3FrameDecoder, V3FrameType} from "../protocol/v3-frame.js"
@@ -22,6 +23,15 @@ import {
   STABLE_IDENTIFIER,
   writeSocket
 } from "./common.js"
+import {
+  connectionFingerprint,
+  createDisconnectAttribution,
+  disconnectReasonForCode,
+  parseDisconnectAttribution,
+  safeSocketCode,
+  sanitizePeerControlDetail,
+  validateFingerprintKeyEpoch
+} from "./disconnect-attribution.js"
 
 const DEFAULT_OUTPUT_BUFFER_BYTES = 32 * 1024 * 1024
 const DEFAULT_RESUME_MS = 60_000
@@ -52,6 +62,8 @@ const MANIFEST_PAGE_STREAMS = 64
  * @property {number} [maxOutputBytesPerConnection] - Per-physical queued hub output bound.
  * @property {number} [resumeWindowMs] - Detached reverse-stream retention window.
  * @property {import('../protocol/compression.js').CompressionLimits} [compressionLimits] - zstd safety bounds.
+ * @property {import('../logging/structured-logger.js').DiagnosticLogger} [logger] - Optional privacy-safe structured logger.
+ * @property {{key: Buffer | Uint8Array, keyEpoch: string}} [connectionFingerprint] - Optional hub-only correlation key and public epoch.
  * @typedef {object} HubPhysical
  * @property {import('node:tls').TLSSocket} socket
  * @property {Principal | undefined} principal
@@ -78,6 +90,10 @@ const MANIFEST_PAGE_STREAMS = 64
  * @property {HubOutputItem[]} outputQueue
  * @property {number} outputBytes
  * @property {boolean} outputWriting
+ * @property {import('./disconnect-attribution.js').DisconnectReason | undefined} firstDisconnectReason
+ * @property {import('./disconnect-attribution.js').DisconnectAttribution | undefined} disconnectAttribution
+ * @property {string | undefined} connectionFingerprint
+ * @property {string | undefined} safeSocketCode
  * @typedef {object} HubOutputItem
  * @property {Buffer} bytes
  * @property {(delivered: boolean) => void} resolve
@@ -119,6 +135,7 @@ const MANIFEST_PAGE_STREAMS = 64
  * @property {{generation: number, clientToGatewaySequence: number, clientToGatewayAck: number, gatewayToClientSequence: number, gatewayToClientAck: number, localEnded: boolean, remoteEnded: boolean} | undefined} clientResume
  * @property {{generation: number, clientToGatewaySequence: number, clientToGatewayAck: number, gatewayToClientSequence: number, gatewayToClientAck: number, localEnded: boolean, remoteEnded: boolean} | undefined} gatewayResume
  * @property {string | undefined} reconciledResume
+ * @property {import('./disconnect-attribution.js').DisconnectAttribution | undefined} disconnectAttribution
  */
 
 /** Starts the Server3 reverse hub. */
@@ -141,6 +158,8 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
   const tcpSockets = new Set()
   /** @type {Map<string, {socket: import('node:net').Socket, timer: ReturnType<typeof setTimeout>}>} */
   const pendingTls = new Map()
+  /** @type {Map<string, {fanOutCount: number, resumeSuccessCount: number, terminalStreamCount: number}>} */
+  const incidents = new Map()
   let bridges = 0
   let physicalAccepts = 0
   let gatewayRegistrations = 0
@@ -209,7 +228,11 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
       clearHandshake: undefined,
       outputQueue: [],
       outputBytes: 0,
-      outputWriting: false
+      outputWriting: false,
+      firstDisconnectReason: undefined,
+      disconnectAttribution: undefined,
+      connectionFingerprint: undefined,
+      safeSocketCode: undefined
     }
     physicals.add(physical)
     const decoder = new V3FrameDecoder({maxFrameBytes: Math.max(MAX_CONTROL_BYTES, maxFrameBytes + 4), maxBufferedBytes: Math.max(MAX_CONTROL_BYTES, maxFrameBytes + 4) * 2})
@@ -232,7 +255,11 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
         if (!socket.destroyed && !physical.closing) socket.resume()
       })
     })
-    socket.on("error", () => {})
+    socket.once("end", () => { physical.firstDisconnectReason ??= "remote_eof" })
+    socket.on("error", (error) => {
+      physical.firstDisconnectReason ??= "remote_socket_error"
+      physical.safeSocketCode ??= safeSocketCode(error)
+    })
     socket.once("close", () => {
       clearTimeout(handshakeTimer)
       void queue.finally(() => detachPhysical(physical, physical.detachCode ?? "TRANSPORT_CLOSED"))
@@ -257,6 +284,7 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
       physical.principal = principal
       physical.connectionId = auth.connectionId
       physical.generation = /** @type {number} */ (auth.generation)
+      if (options.connectionFingerprint !== undefined) physical.connectionFingerprint = connectionFingerprint(options.connectionFingerprint.key, physical.role, physical.connectionId)
       physical.codec = negotiateCodec(auth.compression, availableCodecs())
       if (auth.role === "client") authenticateClientOwner(physical, maxIdentities)
       physical.authenticated = true
@@ -277,6 +305,15 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
     }
     if (frame.type === V3FrameType.PONG) return
     if (frame.type === V3FrameType.GOAWAY) {
+      const detail = decodeV3Control(frame.payload)
+      const sanitized = sanitizePeerControlDetail(detail)
+      if (sanitized.reservedFieldsStripped > 0) logDiagnostic(options.logger, "reverse_hub.reserved_control_field_stripped", {
+        physicalRole: physical.role,
+        ...(physical.role === "client" ? {clientConnectionGeneration: physical.generation} : {gatewayConnectionGeneration: physical.generation}),
+        reservedFieldsStripped: sanitized.reservedFieldsStripped
+      })
+      physical.firstDisconnectReason ??= "peer_goaway"
+      if (typeof sanitized.detail.code === "string") physical.detachCode ??= sanitized.detail.code
       physical.retainOnDetach = false
       physical.socket.end()
       return
@@ -447,7 +484,7 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
         continue
       }
       if (bridge.identifier !== item.identifier) {
-        closeBridge(bridge, "RESUME_IDENTITY_MISMATCH", physical)
+        await closeBridge(bridge, "RESUME_IDENTITY_MISMATCH", physical)
         physical.resumeEntries.push({streamId: item.streamId, bridge: undefined, code: "RESUME_IDENTITY_MISMATCH"})
         continue
       }
@@ -464,7 +501,7 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
     }
     for (const bridge of [...physical.streams.values()]) {
       const streamId = physical.role === "client" ? bridge.clientStreamId : bridge.gatewayStreamId
-      if (!presented.has(streamId)) closeBridge(bridge, "RESUME_STREAM_OMITTED", physical)
+      if (!presented.has(streamId)) await closeBridge(bridge, "RESUME_STREAM_OMITTED", physical)
     }
     physical.resumeItems = []
     physical.resumeFinal = true
@@ -507,7 +544,7 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
           try {
             reconcileBridge(bridge)
           } catch {
-            closeBridge(bridge, "RESUME_STATE_MISMATCH")
+            await closeBridge(bridge, "RESUME_STATE_MISMATCH")
             results.push({streamId: entry.streamId, status: "reset", code: "RESUME_STATE_MISMATCH"})
             continue
           }
@@ -575,6 +612,18 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
     bridge.clientToGatewayCompressor = new AdaptiveCompressor()
     bridge.gatewayToClientCompressor = new AdaptiveCompressor()
     bridge.reconciledResume = key
+    if (bridge.disconnectAttribution !== undefined) {
+      const attribution = bridge.disconnectAttribution
+      const incident = incidentState(attribution)
+      incident.resumeSuccessCount += 1
+      logDiagnostic(options.logger, "reverse_hub.streams_resumed", {
+        ...currentIncidentAttribution(attribution),
+        gatewayId: bridge.gatewayOwner.gatewayId,
+        ...(bridge.client.routeClass === undefined ? {} : {routeClass: bridge.client.routeClass})
+      })
+      bridge.disconnectAttribution = undefined
+      releaseCompletedIncident(attribution)
+    }
   }
 
   /** @param {Record<string, unknown>} value - Resume manifest entry. */
@@ -668,7 +717,8 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
       gatewayToClientCompressor: new AdaptiveCompressor(),
       clientResume: undefined,
       gatewayResume: undefined,
-      reconciledResume: undefined
+      reconciledResume: undefined,
+      disconnectAttribution: undefined
     }
     clientOwner.streams.set(frame.streamId, bridge)
     gatewayOwner.streams.set(gatewayStreamId, bridge)
@@ -711,7 +761,7 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
     }
     if (bridge.state === "closing") {
       if (frame.type === V3FrameType.DATA || frame.type === V3FrameType.END) {
-        closeBridge(bridge, "STREAM_CLOSING")
+        await closeBridge(bridge, "STREAM_CLOSING")
         return
       }
       if (frame.type !== V3FrameType.CLOSE && frame.type !== V3FrameType.RESET) throw protocolError("STREAM_CLOSING")
@@ -742,10 +792,32 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
     if (frame.type === V3FrameType.CLOSE || frame.type === V3FrameType.RESET) {
       const expected = fromClient ? bridge.clientToGatewaySequence : bridge.gatewayToClientSequence
       if ((frame.type === V3FrameType.CLOSE && frame.sequence !== expected) || (frame.type === V3FrameType.RESET && frame.sequence !== 0)) throw protocolError("INVALID_STREAM_CLOSE")
-      const detail = frame.payload.length === 0 ? {} : decodeV3Control(frame.payload)
+      const peerDetail = frame.payload.length === 0 ? {} : decodeV3Control(frame.payload)
+      const sanitized = sanitizePeerControlDetail(peerDetail)
+      if (sanitized.reservedFieldsStripped > 0) logDiagnostic(options.logger, "reverse_hub.reserved_control_field_stripped", {
+        physicalRole: source.role,
+        ...(source.role === "client" ? {clientConnectionGeneration: source.generation} : {gatewayConnectionGeneration: source.generation}),
+        reservedFieldsStripped: sanitized.reservedFieldsStripped
+      })
+      let attribution
+      if (frame.type === V3FrameType.RESET && bridge.disconnectAttribution !== undefined) {
+        incidentState(bridge.disconnectAttribution).terminalStreamCount += 1
+        attribution = currentIncidentAttribution(bridge.disconnectAttribution)
+      }
+      const detail = attribution === undefined ? sanitized.detail : {...sanitized.detail, ...attribution}
       forwardControl(destination, frame.type, destinationStreamId, frame.sequence, detail)
-      if (frame.type === V3FrameType.RESET || fromClient) removeBridge(bridge)
-      else bridge.state = "closing"
+      if (frame.type === V3FrameType.RESET || fromClient) {
+        removeBridge(bridge)
+        if (attribution !== undefined && bridge.disconnectAttribution !== undefined) {
+          logDiagnostic(options.logger, "reverse_hub.streams_reset", {
+            ...attribution,
+            gatewayId: bridge.gatewayOwner.gatewayId,
+            ...(bridge.client.routeClass === undefined ? {} : {routeClass: bridge.client.routeClass}),
+            ...errorDiagnostics(sanitized.detail, "STREAM_RESET")
+          })
+          releaseCompletedIncident(bridge.disconnectAttribution)
+        }
+      } else bridge.state = "closing"
       return
     }
     throw protocolError("UNSUPPORTED_FRAME")
@@ -768,44 +840,76 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
     forwardOutput(destination, encodeV3Frame({type, streamId, sequence}, encodeV3Control(value)))
   }
 
-  /** @param {Bridge} bridge - Stream. @param {string} code - Reset code. @param {HubPhysical} [excluded] - Already-dead peer. */
+  /** @param {Bridge} bridge - Stream. @param {string} code - Reset code. @param {HubPhysical} [excluded] - Already-dead peer. @returns {Promise<void>} Terminal control delivery. */
   function closeBridge(bridge, code, excluded) {
-    if (bridge.client !== excluded && !bridge.client.socket.destroyed) void sendControl(bridge.client, V3FrameType.RESET, bridge.clientStreamId, 0, {code})
-    if (bridge.gateway !== excluded && !bridge.gateway.socket.destroyed) void sendControl(bridge.gateway, V3FrameType.RESET, bridge.gatewayStreamId, 0, {code})
+    const owned = bridgeStillOwned(bridge)
+    let attribution
+    if (owned && bridge.disconnectAttribution !== undefined) {
+      const incident = incidentState(bridge.disconnectAttribution)
+      incident.terminalStreamCount += 1
+      attribution = currentIncidentAttribution(bridge.disconnectAttribution)
+    }
+    const detail = {code, ...(attribution ?? {})}
+    const deliveries = []
+    if (bridge.client !== excluded && !bridge.client.socket.destroyed) deliveries.push(sendControl(bridge.client, V3FrameType.RESET, bridge.clientStreamId, 0, detail))
+    if (bridge.gateway !== excluded && !bridge.gateway.socket.destroyed) deliveries.push(sendControl(bridge.gateway, V3FrameType.RESET, bridge.gatewayStreamId, 0, detail))
     removeBridge(bridge)
+    if (attribution !== undefined) logDiagnostic(options.logger, "reverse_hub.streams_reset", {
+      ...attribution,
+      gatewayId: bridge.gatewayOwner.gatewayId,
+      ...(bridge.client.routeClass === undefined ? {} : {routeClass: bridge.client.routeClass}),
+      ...errorDiagnostics({code})
+    })
+    if (bridge.disconnectAttribution !== undefined) releaseCompletedIncident(bridge.disconnectAttribution)
+    return Promise.all(deliveries).then(() => undefined)
   }
 
-  /** @param {HubPhysical} physical - Connection. @param {string} code - Reason. @param {boolean} [retainStreams] - Preserve logical streams for paired resume. */
-  function terminate(physical, code, retainStreams = false) {
+  /** @param {HubPhysical} physical - Connection. @param {string} code - Reason. @param {boolean} [retainStreams] - Preserve logical streams for paired resume. @param {import('./disconnect-attribution.js').DisconnectAttribution} [attribution] - Propagated first-cause incident. */
+  function terminate(physical, code, retainStreams = false, attribution = undefined) {
     if (physical.closing) return
     physical.closing = true
     physical.retainOnDetach = retainStreams
     physical.detachCode = code
+    const disconnectAttribution = ensurePhysicalAttribution(physical, disconnectReasonForCode(code), attribution)
     const forceTimer = setTimeout(() => physical.socket.destroy(), 1_000)
     forceTimer.unref?.()
-    void sendControl(physical, V3FrameType.GOAWAY, 0, 0, {code}, true).finally(() => {
+    void sendControl(physical, V3FrameType.GOAWAY, 0, 0, {code, ...(disconnectAttribution === undefined ? {} : currentIncidentAttribution(disconnectAttribution))}, true).finally(() => {
       clearTimeout(forceTimer)
       physical.socket.destroy()
     })
   }
 
   /** @param {HubPhysical} physical - Closed connection. @param {string} code - Stream reset. */
-  function detachPhysical(physical, code) {
+  async function detachPhysical(physical, code) {
     if (!physicals.delete(physical)) return
     physical.closing = true
+    const attribution = ensurePhysicalAttribution(physical, physical.firstDisconnectReason ?? disconnectReasonForCode(code))
+    if (attribution !== undefined) logDiagnostic(options.logger, "reverse_hub.physical_detached", {
+      ...currentIncidentAttribution(attribution),
+      ...(physical.connectionFingerprint === undefined || attribution.physicalRole !== physical.role ? {} : {connectionFingerprint: physical.connectionFingerprint, fingerprintKeyEpoch: options.connectionFingerprint?.keyEpoch}),
+      ...(physical.safeSocketCode === undefined ? {} : {safeSocketCode: physical.safeSocketCode})
+    })
     clearResumeTimeout(physical)
     discardQueuedOutput(physical)
     if (physical.gatewayId !== undefined) {
       const owner = gateways.get(physical.gatewayId)
       if (physical.role === "gateway" && owner?.physical === physical) {
         owner.physical = undefined
+        if (attribution !== undefined) for (const bridge of owner.streams.values()) assignBridgeAttribution(bridge, attribution)
         if (!physical.retainOnDetach || closing) {
-          for (const bridge of [...owner.streams.values()]) closeBridge(bridge, code, physical)
+          for (const bridge of [...owner.streams.values()]) await closeBridge(bridge, code, physical)
           gateways.delete(owner.gatewayId)
         } else {
+          if (attribution !== undefined) {
+            logDiagnostic(options.logger, "reverse_hub.streams_interrupted", {...currentIncidentAttribution(attribution), gatewayId: owner.gatewayId})
+          }
           scheduleOwnerExpiry(owner, "gateway")
           const clientsToReconnect = new Set([...owner.streams.values()].map((bridge) => bridge.clientOwner.physical).filter((peer) => peer !== undefined))
-          for (const client of clientsToReconnect) terminate(/** @type {HubPhysical} */ (client), "GATEWAY_DISCONNECTED", true)
+          for (const client of clientsToReconnect) {
+            const peer = /** @type {HubPhysical} */ (client)
+            const propagated = attribution === undefined ? undefined : createDisconnectAttribution({...currentIncidentAttribution(attribution), clientConnectionGeneration: peer.generation})
+            terminate(peer, "GATEWAY_DISCONNECTED", true, propagated)
+          }
         }
       }
     }
@@ -813,37 +917,109 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
       const owner = clients.get(physical.connectionId)
       if (owner?.physical === physical) {
         owner.physical = undefined
+        if (attribution !== undefined) for (const bridge of owner.streams.values()) assignBridgeAttribution(bridge, attribution)
         if (!physical.retainOnDetach || closing) {
-          for (const bridge of [...owner.streams.values()]) closeBridge(bridge, code, physical)
+          for (const bridge of [...owner.streams.values()]) await closeBridge(bridge, code, physical)
           clients.delete(owner.connectionId)
         } else {
+          if (attribution !== undefined) {
+            logDiagnostic(options.logger, "reverse_hub.streams_interrupted", {
+              ...currentIncidentAttribution(attribution),
+              ...(physical.gatewayId === undefined ? {} : {gatewayId: physical.gatewayId}),
+              ...(physical.routeClass === undefined ? {} : {routeClass: physical.routeClass})
+            })
+          }
           scheduleOwnerExpiry(owner, "client")
           const gatewaysToReconnect = new Set([...owner.streams.values()].map((bridge) => bridge.gatewayOwner.physical).filter((peer) => peer !== undefined))
-          for (const gateway of gatewaysToReconnect) terminate(/** @type {HubPhysical} */ (gateway), "CLIENT_DISCONNECTED", true)
+          for (const gateway of gatewaysToReconnect) {
+            const peer = /** @type {HubPhysical} */ (gateway)
+            const propagated = attribution === undefined ? undefined : createDisconnectAttribution({...currentIncidentAttribution(attribution), gatewayConnectionGeneration: peer.generation})
+            terminate(peer, "CLIENT_DISCONNECTED", true, propagated)
+          }
         }
       }
     }
+    if (attribution !== undefined && (!physical.retainOnDetach || incidentState(attribution).fanOutCount === 0)) incidents.delete(attribution.disconnectIncidentId)
+  }
+
+  /** @param {HubPhysical} physical - Connection. @param {import('./disconnect-attribution.js').DisconnectReason} reason - Physical cause. @param {import('./disconnect-attribution.js').DisconnectAttribution | undefined} [propagated] - Existing incident. */
+  function ensurePhysicalAttribution(physical, reason, propagated = undefined) {
+    if (physical.disconnectAttribution !== undefined) return physical.disconnectAttribution
+    if (physical.role === undefined) return undefined
+    const source = propagated === undefined ? undefined : parseDisconnectAttribution(propagated)
+    physical.disconnectAttribution = createDisconnectAttribution({
+      ...(source ?? {}),
+      disconnectReason: source?.disconnectReason ?? reason,
+      incidentOrigin: source?.incidentOrigin ?? "reverse_hub",
+      physicalRole: source?.physicalRole ?? physical.role,
+      ...(physical.role === "client" ? {clientConnectionGeneration: physical.generation} : {gatewayConnectionGeneration: physical.generation}),
+      fanOutCount: source?.fanOutCount ?? physical.streams.size,
+      resumeSuccessCount: source?.resumeSuccessCount ?? 0,
+      terminalStreamCount: source?.terminalStreamCount ?? 0
+    })
+    incidentState(physical.disconnectAttribution)
+    return physical.disconnectAttribution
+  }
+
+  /** Preserves the first incident while completing both endpoint generations. */
+  function assignBridgeAttribution(/** @type {Bridge} */ bridge, /** @type {import('./disconnect-attribution.js').DisconnectAttribution} */ attribution) {
+    bridge.disconnectAttribution ??= createDisconnectAttribution({
+      ...currentIncidentAttribution(attribution),
+      clientConnectionGeneration: bridge.client.generation,
+      gatewayConnectionGeneration: bridge.gateway.generation
+    })
+  }
+
+  /** @param {import('./disconnect-attribution.js').DisconnectAttribution} attribution - Incident. @returns {{fanOutCount: number, resumeSuccessCount: number, terminalStreamCount: number}} */
+  function incidentState(attribution) {
+    let state = incidents.get(attribution.disconnectIncidentId)
+    if (state === undefined) {
+      state = {fanOutCount: attribution.fanOutCount, resumeSuccessCount: attribution.resumeSuccessCount, terminalStreamCount: attribution.terminalStreamCount}
+      incidents.set(attribution.disconnectIncidentId, state)
+    } else {
+      state.fanOutCount = Math.max(state.fanOutCount, attribution.fanOutCount)
+      state.resumeSuccessCount = Math.max(state.resumeSuccessCount, attribution.resumeSuccessCount)
+      state.terminalStreamCount = Math.max(state.terminalStreamCount, attribution.terminalStreamCount)
+    }
+    return state
+  }
+
+  /** @param {import('./disconnect-attribution.js').DisconnectAttribution} attribution - Incident. */
+  function currentIncidentAttribution(attribution) {
+    const state = incidentState(attribution)
+    return createDisconnectAttribution({...attribution, ...state})
+  }
+
+  /** @param {import('./disconnect-attribution.js').DisconnectAttribution} attribution - Incident. */
+  function releaseCompletedIncident(attribution) {
+    const state = incidentState(attribution)
+    if (state.resumeSuccessCount + state.terminalStreamCount >= state.fanOutCount) incidents.delete(attribution.disconnectIncidentId)
   }
 
   /** @param {GatewayOwner | ClientOwner} owner - Detached identity. @param {'gateway' | 'client'} role - Owner role. */
   function scheduleOwnerExpiry(owner, role) {
     if (owner.expiryTimer !== undefined) clearTimeout(owner.expiryTimer)
     owner.expiryTimer = setTimeout(() => {
-      owner.expiryTimer = undefined
-      if (owner.physical !== undefined) return
-      for (const bridge of [...owner.streams.values()]) closeBridge(bridge, "RESUME_WINDOW_EXPIRED")
-      if (role === "gateway") {
-        const gatewayOwner = /** @type {GatewayOwner} */ (owner)
-        if (gateways.get(gatewayOwner.gatewayId) === gatewayOwner) gateways.delete(gatewayOwner.gatewayId)
-      } else {
-        const clientOwner = /** @type {ClientOwner} */ (owner)
-        if (clients.get(clientOwner.connectionId) === clientOwner) clients.delete(clientOwner.connectionId)
-      }
-      void tryCompleteResumes()
+      void expireOwner(owner, role)
     }, resumeWindowMs)
     owner.expiryTimer.unref?.()
   }
 
+  /** @param {GatewayOwner | ClientOwner} owner - Detached identity. @param {'gateway' | 'client'} role - Owner role. */
+  async function expireOwner(owner, role) {
+    owner.expiryTimer = undefined
+    if (owner.physical !== undefined) return
+    for (const bridge of [...owner.streams.values()]) await closeBridge(bridge, "RESUME_WINDOW_EXPIRED")
+    if (role === "gateway") {
+      const gatewayOwner = /** @type {GatewayOwner} */ (owner)
+      if (gateways.get(gatewayOwner.gatewayId) === gatewayOwner) gateways.delete(gatewayOwner.gatewayId)
+    } else {
+      const clientOwner = /** @type {ClientOwner} */ (owner)
+      if (clients.get(clientOwner.connectionId) === clientOwner) clients.delete(clientOwner.connectionId)
+    }
+    void tryCompleteResumes()
+  }
+
   /** @param {HubPhysical} physical - Destination. @param {Buffer} bytes - Encoded frame. @param {boolean} [allowClosing] - Allow a final GOAWAY after closing starts. @returns {Promise<boolean>} Delivery result. */
   function enqueueOutput(physical, bytes, allowClosing = false) {
     if (physical.socket.destroyed || (physical.closing && !allowClosing)) return Promise.resolve(false)
@@ -897,6 +1073,8 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
     physical.closing = true
     physical.retainOnDetach = false
     physical.detachCode ??= code
+    physical.firstDisconnectReason ??= disconnectReasonForCode(code)
+    ensurePhysicalAttribution(physical, physical.firstDisconnectReason)
     discardQueuedOutput(physical)
     if (!physical.socket.destroyed) physical.socket.destroy(protocolError(code))
   }
@@ -961,6 +1139,11 @@ function socketKey(socket) {
 /** @param {ReverseGatewayHubOptions} options - Options. */
 function validateOptions(options) {
   if (typeof options !== "object" || options === null || options.tokens === undefined || typeof options.tokens.authenticate !== "function") throw new TypeError("Reverse gateway hub requires a token authenticator")
+  if (options.connectionFingerprint !== undefined) {
+    const {key, keyEpoch} = options.connectionFingerprint
+    if (!(key instanceof Uint8Array) || key.byteLength < 32) throw new TypeError("connection fingerprint key must contain at least 32 bytes")
+    validateFingerprintKeyEpoch(keyEpoch)
+  }
   if (typeof options.authorizeGateway !== "function" || typeof options.authorizeClientGateway !== "function" || typeof options.authorizeOpen !== "function") throw new TypeError("Reverse gateway hub requires gateway, client selection, and open authorization policies")
   if (typeof options.tls !== "object" || options.tls === null || typeof options.tls.certFile !== "string" || typeof options.tls.keyFile !== "string") throw new TypeError("Reverse gateway hub requires TLS certFile and keyFile")
   if (!zstdAvailable()) throw new Error("Reverse gateway hub requires Node.js zstd support")

package/src/reverse/runtime.js

@@ -3,8 +3,8 @@ import {StaticTokenStore} from "../auth/static-token-store.js"
 import {OutboundReverseGateway} from "./gateway.js"
 import {startReverseGatewayHub} from "./hub.js"
 
-/** @param {import('../config/environment.js').RelayEnvironmentConfig} config - Parsed environment. */
-export async function startReverseHubRuntime(config) {
+/** @param {import('../config/environment.js').RelayEnvironmentConfig} config - Parsed environment. @param {import('../logging/structured-logger.js').DiagnosticLogger} [logger] - Optional structured logger. */
+export async function startReverseHubRuntime(config, logger) {
   if (config.mode !== "reverse-hub") throw new Error("Reverse hub runtime requires SOCKETDUCT_MODE=reverse-hub")
   if (config.relayTls === undefined) throw new Error("Reverse hub mode requires SOCKETDUCT_TLS_CERT_FILE and SOCKETDUCT_TLS_KEY_FILE")
   if (config.authTokens.length === 0) throw new Error("SOCKETDUCT_AUTH_TOKENS must contain hub client and gateway credentials")
@@ -54,7 +54,9 @@ export async function startReverseHubRuntime(config) {
     resumeWindowMs: config.resumeWindowMs,
     handshakeTimeoutMs: config.handshakeTimeoutMs,
     maxInputBytesPerConnection: config.maxInputBytesPerConnection,
-    compressionLimits: compressionLimits(config)
+    compressionLimits: compressionLimits(config),
+    ...(logger === undefined ? {} : {logger}),
+    ...(config.connectionFingerprint === undefined ? {} : {connectionFingerprint: config.connectionFingerprint})
   })
   return {hub, close: hub.close}
 }

package/src/spool/spool-file.js

@@ -18,6 +18,12 @@ import {join, resolve} from "node:path"
  * @property {string} [ownerId] - ownerId value.
  * @property {number} sequence - sequence value.
  * @property {Uint8Array} data - data value.
+ * @typedef {object} AppendChunkBatchInput
+ * @property {string} sessionId - Stable session identifier.
+ * @property {string} ownerId - Expected spool owner.
+ * @property {number} previousSequence - Last sequence committed by recovery metadata.
+ * @property {readonly SpoolChunk[]} chunks - Complete application operation split into frames.
+ * @property {Record<string, unknown>} recoveryState - Recovery commit written after every chunk.
  * @typedef {object} CreateSpoolInput
  * @property {string} sessionId - sessionId value.
  * @property {string} ownerId - ownerId value.
@@ -125,6 +131,23 @@ export class SessionSpool {
     }
   }
 
+  /**
+   * Looks up recovery state without treating an absent session as corruption.
+   * @param {string} sessionId - Stable session identifier.
+   * @param {string} ownerId - Expected spool owner.
+   * @returns {Record<string, unknown> | undefined} Parsed state, or undefined when the session directory is absent.
+   */
+  findRecoveryStateSync(sessionId, ownerId) {
+    const directory = this.#sessionDirectory(sessionId)
+    try {
+      statSync(directory)
+    } catch (error) {
+      if (/** @type {{code?: unknown}} */ (error).code === "ENOENT") return undefined
+      throw error
+    }
+    return this.readRecoveryStateSync(sessionId, ownerId)
+  }
+
   /**
    * Loads durable chunks synchronously so recovery can fail before network
    * activity. Ownership, age, encoding, accounting, and configured byte bounds
@@ -180,13 +203,15 @@ export class SessionSpool {
         if (usage.totalBytes !== totalBytes) throw new Error("recovery chunk accounting mismatch")
         this.#usedBytes.set(sessionId, totalBytes)
         this.#usageLoaded.add(sessionId)
-        return chunks.filter((chunk) => chunk.sequence > acknowledgedSequence)
+        return chunksAfterSequence(chunks, acknowledgedSequence)
       }
-      if (!committedChunksAreCovered(chunks, acknowledgedSequence, committedSequence)) throw new Error("recovery chunks do not cover the committed sequence")
-      const firstAfterAck = chunks.find((chunk) => chunk.sequence > acknowledgedSequence)
+      const unacknowledged = chunksAfterSequence(chunks, acknowledgedSequence)
+      if (!committedChunksAreCovered(unacknowledged, acknowledgedSequence, committedSequence)) throw new Error("recovery chunks do not cover the committed sequence")
+      const firstAfterAck = unacknowledged[0]
       if (firstAfterAck !== undefined && firstAfterAck.sequence - firstAfterAck.data.length !== acknowledgedSequence) throw new Error("recovery chunks do not start at the acknowledgement")
-      if (firstAfterAck === undefined && chunks.length > 0 && chunks.at(-1)?.sequence !== acknowledgedSequence) throw new Error("recovery acknowledgement is not a chunk boundary")
-      const committed = chunks.filter((chunk) => chunk.sequence > acknowledgedSequence && chunk.sequence <= committedSequence)
+      const lastChunk = chunks.at(-1)
+      if (firstAfterAck === undefined && lastChunk !== undefined && lastChunk.sequence < acknowledgedSequence) throw new Error("recovery acknowledgement exceeds stored chunks")
+      const committed = unacknowledged.filter((chunk) => chunk.sequence <= committedSequence)
       const committedBytes = committed.reduce((sum, chunk) => sum + chunk.data.length, 0)
       if (!Number.isSafeInteger(committedBytes) || (this.#maxBytes !== undefined && committedBytes > this.#maxBytes)) throw new Error("recovery chunks exceed the configured byte limit")
       const repairedChunks = serializeChunks(committed)
@@ -360,6 +385,45 @@ export class SessionSpool {
     this.#usedBytes.set(input.sessionId, newTotal)
   }
 
+  /**
+   * Replaces any uncommitted tail, writes a complete operation, then advances
+   * recovery metadata as the commit record. A failure before the last step is
+   * ignored by recovery and safely replaced by a retry.
+   * @param {AppendChunkBatchInput} input - Atomic durable admission input.
+   * @returns {Promise<void>} Completion after the recovery commit.
+   */
+  async appendChunksAndRecoveryState(input) {
+    const session = await this.#readSession(input.sessionId)
+    if (session.metadata.ownerId !== input.ownerId) throw new Error("Chunk owner does not match spool metadata")
+    if (!Number.isSafeInteger(input.previousSequence) || input.previousSequence < 0) throw new RangeError("Previous durable sequence must be a non-negative safe integer")
+    if (!Array.isArray(input.chunks) || input.chunks.length === 0) throw new TypeError("Durable chunk batch must not be empty")
+    const encodedRecovery = `${JSON.stringify(input.recoveryState)}\n`
+    if (Buffer.byteLength(encodedRecovery) > MAX_RECOVERY_STATE_BYTES) throw new Error("Socketduct recovery state exceeds its size limit")
+
+    const createdAt = new Date(session.metadata.createdAt).getTime()
+    const ageMs = this.#now() - createdAt
+    if (this.#maxAgeMs !== undefined && ageMs > this.#maxAgeMs) throw new SpoolQuotaError(`Spool age ${ageMs}ms exceeds max ${this.#maxAgeMs}ms for session ${input.sessionId}`, "SPOOL_AGE_EXCEEDED")
+
+    const stored = await this.#readAllChunks(input.sessionId)
+    const committed = stored.filter((chunk) => chunk.sequence <= input.previousSequence)
+    let sequence = input.previousSequence
+    const additions = input.chunks.map((chunk) => {
+      const data = Buffer.from(chunk.data)
+      if (data.length === 0 || !Number.isSafeInteger(chunk.sequence) || chunk.sequence !== sequence + data.length) throw new Error("Durable chunk batch is not contiguous")
+      sequence = chunk.sequence
+      return {sequence: chunk.sequence, data}
+    })
+    const chunks = [...committed, ...additions]
+    const totalBytes = chunks.reduce((sum, chunk) => sum + chunk.data.length, 0)
+    if (!Number.isSafeInteger(totalBytes) || (this.#maxBytes !== undefined && totalBytes > this.#maxBytes)) throw new SpoolQuotaError(`Spool byte quota exceeded for session ${input.sessionId}: ${totalBytes} / max ${this.#maxBytes}`, "SPOOL_QUOTA_EXCEEDED")
+
+    await this.#writeChunks(input.sessionId, chunks)
+    await writeFileAtomic(join(session.directory, "usage.json"), JSON.stringify({totalBytes}) + "\n", {mode: 0o600})
+    this.#usedBytes.set(input.sessionId, totalBytes)
+    this.#usageLoaded.add(input.sessionId)
+    await this.writeRecoveryState(input.sessionId, input.recoveryState)
+  }
+
   /**
    * @param {string} sessionId - sessionId value.
    * @param {number} acknowledgedSequence - acknowledgedSequence value.
@@ -377,7 +441,8 @@ export class SessionSpool {
    */
   async truncateAcked(sessionId, acknowledgedSequence) {
     const chunks = await this.#readAllChunks(sessionId)
-    const remaining = chunks.filter((chunk) => chunk.sequence > acknowledgedSequence)
+    const byteSequenced = chunks.every((chunk, index) => chunk.data.length <= chunk.sequence && (index === 0 || chunk.sequence - chunk.data.length === chunks[index - 1]?.sequence))
+    const remaining = byteSequenced ? chunksAfterSequence(chunks, acknowledgedSequence) : chunks.filter((chunk) => chunk.sequence > acknowledgedSequence)
     await this.#writeChunks(sessionId, remaining)
     const newTotal = remaining.reduce((sum, chunk) => sum + chunk.data.length, 0)
     const directory = this.#sessionDirectory(sessionId)
@@ -547,6 +612,27 @@ function serializeChunks(chunks) {
   return lines.length === 0 ? "" : `${lines.join("\n")}\n`
 }
 
+/**
+ * Removes an acknowledged prefix, including a byte-granular prefix inside the
+ * first retained physical chunk.
+ * @param {readonly SpoolChunk[]} chunks - Ordered chunks.
+ * @param {number} acknowledgedSequence - Cumulative application-byte ACK.
+ * @returns {SpoolChunk[]} Unacknowledged suffix.
+ */
+function chunksAfterSequence(chunks, acknowledgedSequence) {
+  /** @type {SpoolChunk[]} */
+  const remaining = []
+  for (const chunk of chunks) {
+    if (chunk.sequence <= acknowledgedSequence) continue
+    const start = chunk.sequence - chunk.data.length
+    remaining.push({
+      sequence: chunk.sequence,
+      data: start < acknowledgedSequence ? Buffer.from(chunk.data.subarray(acknowledgedSequence - start)) : Buffer.from(chunk.data)
+    })
+  }
+  return remaining
+}
+
 /**
  * @param {readonly SpoolChunk[]} chunks - Structurally validated physical chunks.
  * @param {number} acknowledgedSequence - Durable ACK boundary.

package/tensorbuzz.yml

@@ -2,7 +2,7 @@ dependency_roots:
   - path: .
 before_install:
   - curl -fsSL https://deb.nodesource.com/setup_24.x | sudo -E bash -
-  - sudo apt-get install -y docker.io nodejs
+  - sudo apt-get install -y docker.io nodejs zstd
 before_script:
   - node --version
 environment:
@@ -34,4 +34,5 @@ script:
   - npm run typecheck
   - npm run lint
   - npm test
+  - npm run verify:zstd-logs
   - docker build --tag socketduct-ci .