socketduct 0.1.19 → 0.1.20

package/src/reverse/hub.js

@@ -2,6 +2,7 @@
 import {readFileSync} from "node:fs"
 import {createServer as createTlsServer} from "node:tls"
 
+import {errorDiagnostics, logDiagnostic} from "../logging/structured-logger.js"
 import {availableCodecs, negotiateCodec, zstdAvailable} from "../protocol/compression.js"
 import {decodeV3Data, encodeV3Data, AdaptiveCompressor} from "../protocol/v3-compression.js"
 import {decodeV3Control, encodeV3Control, encodeV3Frame, V3FrameDecoder, V3FrameType} from "../protocol/v3-frame.js"
@@ -22,6 +23,15 @@ import {
   STABLE_IDENTIFIER,
   writeSocket
 } from "./common.js"
+import {
+  connectionFingerprint,
+  createDisconnectAttribution,
+  disconnectReasonForCode,
+  parseDisconnectAttribution,
+  safeSocketCode,
+  sanitizePeerControlDetail,
+  validateFingerprintKeyEpoch
+} from "./disconnect-attribution.js"
 
 const DEFAULT_OUTPUT_BUFFER_BYTES = 32 * 1024 * 1024
 const DEFAULT_RESUME_MS = 60_000
@@ -52,6 +62,8 @@ const MANIFEST_PAGE_STREAMS = 64
  * @property {number} [maxOutputBytesPerConnection] - Per-physical queued hub output bound.
  * @property {number} [resumeWindowMs] - Detached reverse-stream retention window.
  * @property {import('../protocol/compression.js').CompressionLimits} [compressionLimits] - zstd safety bounds.
+ * @property {import('../logging/structured-logger.js').DiagnosticLogger} [logger] - Optional privacy-safe structured logger.
+ * @property {{key: Buffer | Uint8Array, keyEpoch: string}} [connectionFingerprint] - Optional hub-only correlation key and public epoch.
  * @typedef {object} HubPhysical
  * @property {import('node:tls').TLSSocket} socket
  * @property {Principal | undefined} principal
@@ -78,6 +90,10 @@ const MANIFEST_PAGE_STREAMS = 64
  * @property {HubOutputItem[]} outputQueue
  * @property {number} outputBytes
  * @property {boolean} outputWriting
+ * @property {import('./disconnect-attribution.js').DisconnectReason | undefined} firstDisconnectReason
+ * @property {import('./disconnect-attribution.js').DisconnectAttribution | undefined} disconnectAttribution
+ * @property {string | undefined} connectionFingerprint
+ * @property {string | undefined} safeSocketCode
  * @typedef {object} HubOutputItem
  * @property {Buffer} bytes
  * @property {(delivered: boolean) => void} resolve
@@ -119,6 +135,7 @@ const MANIFEST_PAGE_STREAMS = 64
  * @property {{generation: number, clientToGatewaySequence: number, clientToGatewayAck: number, gatewayToClientSequence: number, gatewayToClientAck: number, localEnded: boolean, remoteEnded: boolean} | undefined} clientResume
  * @property {{generation: number, clientToGatewaySequence: number, clientToGatewayAck: number, gatewayToClientSequence: number, gatewayToClientAck: number, localEnded: boolean, remoteEnded: boolean} | undefined} gatewayResume
  * @property {string | undefined} reconciledResume
+ * @property {import('./disconnect-attribution.js').DisconnectAttribution | undefined} disconnectAttribution
  */
 
 /** Starts the Server3 reverse hub. */
@@ -141,6 +158,8 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
   const tcpSockets = new Set()
   /** @type {Map<string, {socket: import('node:net').Socket, timer: ReturnType<typeof setTimeout>}>} */
   const pendingTls = new Map()
+  /** @type {Map<string, {fanOutCount: number, resumeSuccessCount: number, terminalStreamCount: number}>} */
+  const incidents = new Map()
   let bridges = 0
   let physicalAccepts = 0
   let gatewayRegistrations = 0
@@ -209,7 +228,11 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
       clearHandshake: undefined,
       outputQueue: [],
       outputBytes: 0,
-      outputWriting: false
+      outputWriting: false,
+      firstDisconnectReason: undefined,
+      disconnectAttribution: undefined,
+      connectionFingerprint: undefined,
+      safeSocketCode: undefined
     }
     physicals.add(physical)
     const decoder = new V3FrameDecoder({maxFrameBytes: Math.max(MAX_CONTROL_BYTES, maxFrameBytes + 4), maxBufferedBytes: Math.max(MAX_CONTROL_BYTES, maxFrameBytes + 4) * 2})
@@ -232,7 +255,11 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
         if (!socket.destroyed && !physical.closing) socket.resume()
       })
     })
-    socket.on("error", () => {})
+    socket.once("end", () => { physical.firstDisconnectReason ??= "remote_eof" })
+    socket.on("error", (error) => {
+      physical.firstDisconnectReason ??= "remote_socket_error"
+      physical.safeSocketCode ??= safeSocketCode(error)
+    })
     socket.once("close", () => {
       clearTimeout(handshakeTimer)
       void queue.finally(() => detachPhysical(physical, physical.detachCode ?? "TRANSPORT_CLOSED"))
@@ -257,6 +284,7 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
       physical.principal = principal
       physical.connectionId = auth.connectionId
       physical.generation = /** @type {number} */ (auth.generation)
+      if (options.connectionFingerprint !== undefined) physical.connectionFingerprint = connectionFingerprint(options.connectionFingerprint.key, physical.role, physical.connectionId)
       physical.codec = negotiateCodec(auth.compression, availableCodecs())
       if (auth.role === "client") authenticateClientOwner(physical, maxIdentities)
       physical.authenticated = true
@@ -277,6 +305,15 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
     }
     if (frame.type === V3FrameType.PONG) return
     if (frame.type === V3FrameType.GOAWAY) {
+      const detail = decodeV3Control(frame.payload)
+      const sanitized = sanitizePeerControlDetail(detail)
+      if (sanitized.reservedFieldsStripped > 0) logDiagnostic(options.logger, "reverse_hub.reserved_control_field_stripped", {
+        physicalRole: physical.role,
+        ...(physical.role === "client" ? {clientConnectionGeneration: physical.generation} : {gatewayConnectionGeneration: physical.generation}),
+        reservedFieldsStripped: sanitized.reservedFieldsStripped
+      })
+      physical.firstDisconnectReason ??= "peer_goaway"
+      if (typeof sanitized.detail.code === "string") physical.detachCode ??= sanitized.detail.code
       physical.retainOnDetach = false
       physical.socket.end()
       return
@@ -447,7 +484,7 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
         continue
       }
       if (bridge.identifier !== item.identifier) {
-        closeBridge(bridge, "RESUME_IDENTITY_MISMATCH", physical)
+        await closeBridge(bridge, "RESUME_IDENTITY_MISMATCH", physical)
         physical.resumeEntries.push({streamId: item.streamId, bridge: undefined, code: "RESUME_IDENTITY_MISMATCH"})
         continue
       }
@@ -464,7 +501,7 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
     }
     for (const bridge of [...physical.streams.values()]) {
       const streamId = physical.role === "client" ? bridge.clientStreamId : bridge.gatewayStreamId
-      if (!presented.has(streamId)) closeBridge(bridge, "RESUME_STREAM_OMITTED", physical)
+      if (!presented.has(streamId)) await closeBridge(bridge, "RESUME_STREAM_OMITTED", physical)
     }
     physical.resumeItems = []
     physical.resumeFinal = true
@@ -507,7 +544,7 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
           try {
             reconcileBridge(bridge)
           } catch {
-            closeBridge(bridge, "RESUME_STATE_MISMATCH")
+            await closeBridge(bridge, "RESUME_STATE_MISMATCH")
             results.push({streamId: entry.streamId, status: "reset", code: "RESUME_STATE_MISMATCH"})
             continue
           }
@@ -575,6 +612,18 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
     bridge.clientToGatewayCompressor = new AdaptiveCompressor()
     bridge.gatewayToClientCompressor = new AdaptiveCompressor()
     bridge.reconciledResume = key
+    if (bridge.disconnectAttribution !== undefined) {
+      const attribution = bridge.disconnectAttribution
+      const incident = incidentState(attribution)
+      incident.resumeSuccessCount += 1
+      logDiagnostic(options.logger, "reverse_hub.streams_resumed", {
+        ...currentIncidentAttribution(attribution),
+        gatewayId: bridge.gatewayOwner.gatewayId,
+        ...(bridge.client.routeClass === undefined ? {} : {routeClass: bridge.client.routeClass})
+      })
+      bridge.disconnectAttribution = undefined
+      releaseCompletedIncident(attribution)
+    }
   }
 
   /** @param {Record<string, unknown>} value - Resume manifest entry. */
@@ -668,7 +717,8 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
       gatewayToClientCompressor: new AdaptiveCompressor(),
       clientResume: undefined,
       gatewayResume: undefined,
-      reconciledResume: undefined
+      reconciledResume: undefined,
+      disconnectAttribution: undefined
     }
     clientOwner.streams.set(frame.streamId, bridge)
     gatewayOwner.streams.set(gatewayStreamId, bridge)
@@ -711,7 +761,7 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
     }
     if (bridge.state === "closing") {
       if (frame.type === V3FrameType.DATA || frame.type === V3FrameType.END) {
-        closeBridge(bridge, "STREAM_CLOSING")
+        await closeBridge(bridge, "STREAM_CLOSING")
         return
       }
       if (frame.type !== V3FrameType.CLOSE && frame.type !== V3FrameType.RESET) throw protocolError("STREAM_CLOSING")
@@ -742,10 +792,32 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
     if (frame.type === V3FrameType.CLOSE || frame.type === V3FrameType.RESET) {
       const expected = fromClient ? bridge.clientToGatewaySequence : bridge.gatewayToClientSequence
       if ((frame.type === V3FrameType.CLOSE && frame.sequence !== expected) || (frame.type === V3FrameType.RESET && frame.sequence !== 0)) throw protocolError("INVALID_STREAM_CLOSE")
-      const detail = frame.payload.length === 0 ? {} : decodeV3Control(frame.payload)
+      const peerDetail = frame.payload.length === 0 ? {} : decodeV3Control(frame.payload)
+      const sanitized = sanitizePeerControlDetail(peerDetail)
+      if (sanitized.reservedFieldsStripped > 0) logDiagnostic(options.logger, "reverse_hub.reserved_control_field_stripped", {
+        physicalRole: source.role,
+        ...(source.role === "client" ? {clientConnectionGeneration: source.generation} : {gatewayConnectionGeneration: source.generation}),
+        reservedFieldsStripped: sanitized.reservedFieldsStripped
+      })
+      let attribution
+      if (frame.type === V3FrameType.RESET && bridge.disconnectAttribution !== undefined) {
+        incidentState(bridge.disconnectAttribution).terminalStreamCount += 1
+        attribution = currentIncidentAttribution(bridge.disconnectAttribution)
+      }
+      const detail = attribution === undefined ? sanitized.detail : {...sanitized.detail, ...attribution}
       forwardControl(destination, frame.type, destinationStreamId, frame.sequence, detail)
-      if (frame.type === V3FrameType.RESET || fromClient) removeBridge(bridge)
-      else bridge.state = "closing"
+      if (frame.type === V3FrameType.RESET || fromClient) {
+        removeBridge(bridge)
+        if (attribution !== undefined && bridge.disconnectAttribution !== undefined) {
+          logDiagnostic(options.logger, "reverse_hub.streams_reset", {
+            ...attribution,
+            gatewayId: bridge.gatewayOwner.gatewayId,
+            ...(bridge.client.routeClass === undefined ? {} : {routeClass: bridge.client.routeClass}),
+            ...errorDiagnostics(sanitized.detail, "STREAM_RESET")
+          })
+          releaseCompletedIncident(bridge.disconnectAttribution)
+        }
+      } else bridge.state = "closing"
       return
     }
     throw protocolError("UNSUPPORTED_FRAME")
@@ -768,44 +840,76 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
     forwardOutput(destination, encodeV3Frame({type, streamId, sequence}, encodeV3Control(value)))
   }
 
-  /** @param {Bridge} bridge - Stream. @param {string} code - Reset code. @param {HubPhysical} [excluded] - Already-dead peer. */
+  /** @param {Bridge} bridge - Stream. @param {string} code - Reset code. @param {HubPhysical} [excluded] - Already-dead peer. @returns {Promise<void>} Terminal control delivery. */
   function closeBridge(bridge, code, excluded) {
-    if (bridge.client !== excluded && !bridge.client.socket.destroyed) void sendControl(bridge.client, V3FrameType.RESET, bridge.clientStreamId, 0, {code})
-    if (bridge.gateway !== excluded && !bridge.gateway.socket.destroyed) void sendControl(bridge.gateway, V3FrameType.RESET, bridge.gatewayStreamId, 0, {code})
+    const owned = bridgeStillOwned(bridge)
+    let attribution
+    if (owned && bridge.disconnectAttribution !== undefined) {
+      const incident = incidentState(bridge.disconnectAttribution)
+      incident.terminalStreamCount += 1
+      attribution = currentIncidentAttribution(bridge.disconnectAttribution)
+    }
+    const detail = {code, ...(attribution ?? {})}
+    const deliveries = []
+    if (bridge.client !== excluded && !bridge.client.socket.destroyed) deliveries.push(sendControl(bridge.client, V3FrameType.RESET, bridge.clientStreamId, 0, detail))
+    if (bridge.gateway !== excluded && !bridge.gateway.socket.destroyed) deliveries.push(sendControl(bridge.gateway, V3FrameType.RESET, bridge.gatewayStreamId, 0, detail))
     removeBridge(bridge)
+    if (attribution !== undefined) logDiagnostic(options.logger, "reverse_hub.streams_reset", {
+      ...attribution,
+      gatewayId: bridge.gatewayOwner.gatewayId,
+      ...(bridge.client.routeClass === undefined ? {} : {routeClass: bridge.client.routeClass}),
+      ...errorDiagnostics({code})
+    })
+    if (bridge.disconnectAttribution !== undefined) releaseCompletedIncident(bridge.disconnectAttribution)
+    return Promise.all(deliveries).then(() => undefined)
   }
 
-  /** @param {HubPhysical} physical - Connection. @param {string} code - Reason. @param {boolean} [retainStreams] - Preserve logical streams for paired resume. */
-  function terminate(physical, code, retainStreams = false) {
+  /** @param {HubPhysical} physical - Connection. @param {string} code - Reason. @param {boolean} [retainStreams] - Preserve logical streams for paired resume. @param {import('./disconnect-attribution.js').DisconnectAttribution} [attribution] - Propagated first-cause incident. */
+  function terminate(physical, code, retainStreams = false, attribution = undefined) {
     if (physical.closing) return
     physical.closing = true
     physical.retainOnDetach = retainStreams
     physical.detachCode = code
+    const disconnectAttribution = ensurePhysicalAttribution(physical, disconnectReasonForCode(code), attribution)
     const forceTimer = setTimeout(() => physical.socket.destroy(), 1_000)
     forceTimer.unref?.()
-    void sendControl(physical, V3FrameType.GOAWAY, 0, 0, {code}, true).finally(() => {
+    void sendControl(physical, V3FrameType.GOAWAY, 0, 0, {code, ...(disconnectAttribution === undefined ? {} : currentIncidentAttribution(disconnectAttribution))}, true).finally(() => {
       clearTimeout(forceTimer)
       physical.socket.destroy()
     })
   }
 
   /** @param {HubPhysical} physical - Closed connection. @param {string} code - Stream reset. */
-  function detachPhysical(physical, code) {
+  async function detachPhysical(physical, code) {
     if (!physicals.delete(physical)) return
     physical.closing = true
+    const attribution = ensurePhysicalAttribution(physical, physical.firstDisconnectReason ?? disconnectReasonForCode(code))
+    if (attribution !== undefined) logDiagnostic(options.logger, "reverse_hub.physical_detached", {
+      ...currentIncidentAttribution(attribution),
+      ...(physical.connectionFingerprint === undefined || attribution.physicalRole !== physical.role ? {} : {connectionFingerprint: physical.connectionFingerprint, fingerprintKeyEpoch: options.connectionFingerprint?.keyEpoch}),
+      ...(physical.safeSocketCode === undefined ? {} : {safeSocketCode: physical.safeSocketCode})
+    })
     clearResumeTimeout(physical)
     discardQueuedOutput(physical)
     if (physical.gatewayId !== undefined) {
       const owner = gateways.get(physical.gatewayId)
       if (physical.role === "gateway" && owner?.physical === physical) {
         owner.physical = undefined
+        if (attribution !== undefined) for (const bridge of owner.streams.values()) assignBridgeAttribution(bridge, attribution)
         if (!physical.retainOnDetach || closing) {
-          for (const bridge of [...owner.streams.values()]) closeBridge(bridge, code, physical)
+          for (const bridge of [...owner.streams.values()]) await closeBridge(bridge, code, physical)
           gateways.delete(owner.gatewayId)
         } else {
+          if (attribution !== undefined) {
+            logDiagnostic(options.logger, "reverse_hub.streams_interrupted", {...currentIncidentAttribution(attribution), gatewayId: owner.gatewayId})
+          }
           scheduleOwnerExpiry(owner, "gateway")
           const clientsToReconnect = new Set([...owner.streams.values()].map((bridge) => bridge.clientOwner.physical).filter((peer) => peer !== undefined))
-          for (const client of clientsToReconnect) terminate(/** @type {HubPhysical} */ (client), "GATEWAY_DISCONNECTED", true)
+          for (const client of clientsToReconnect) {
+            const peer = /** @type {HubPhysical} */ (client)
+            const propagated = attribution === undefined ? undefined : createDisconnectAttribution({...currentIncidentAttribution(attribution), clientConnectionGeneration: peer.generation})
+            terminate(peer, "GATEWAY_DISCONNECTED", true, propagated)
+          }
         }
       }
     }
@@ -813,37 +917,109 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
       const owner = clients.get(physical.connectionId)
       if (owner?.physical === physical) {
         owner.physical = undefined
+        if (attribution !== undefined) for (const bridge of owner.streams.values()) assignBridgeAttribution(bridge, attribution)
         if (!physical.retainOnDetach || closing) {
-          for (const bridge of [...owner.streams.values()]) closeBridge(bridge, code, physical)
+          for (const bridge of [...owner.streams.values()]) await closeBridge(bridge, code, physical)
           clients.delete(owner.connectionId)
         } else {
+          if (attribution !== undefined) {
+            logDiagnostic(options.logger, "reverse_hub.streams_interrupted", {
+              ...currentIncidentAttribution(attribution),
+              ...(physical.gatewayId === undefined ? {} : {gatewayId: physical.gatewayId}),
+              ...(physical.routeClass === undefined ? {} : {routeClass: physical.routeClass})
+            })
+          }
           scheduleOwnerExpiry(owner, "client")
           const gatewaysToReconnect = new Set([...owner.streams.values()].map((bridge) => bridge.gatewayOwner.physical).filter((peer) => peer !== undefined))
-          for (const gateway of gatewaysToReconnect) terminate(/** @type {HubPhysical} */ (gateway), "CLIENT_DISCONNECTED", true)
+          for (const gateway of gatewaysToReconnect) {
+            const peer = /** @type {HubPhysical} */ (gateway)
+            const propagated = attribution === undefined ? undefined : createDisconnectAttribution({...currentIncidentAttribution(attribution), gatewayConnectionGeneration: peer.generation})
+            terminate(peer, "CLIENT_DISCONNECTED", true, propagated)
+          }
         }
       }
     }
+    if (attribution !== undefined && (!physical.retainOnDetach || incidentState(attribution).fanOutCount === 0)) incidents.delete(attribution.disconnectIncidentId)
+  }
+
+  /** @param {HubPhysical} physical - Connection. @param {import('./disconnect-attribution.js').DisconnectReason} reason - Physical cause. @param {import('./disconnect-attribution.js').DisconnectAttribution | undefined} [propagated] - Existing incident. */
+  function ensurePhysicalAttribution(physical, reason, propagated = undefined) {
+    if (physical.disconnectAttribution !== undefined) return physical.disconnectAttribution
+    if (physical.role === undefined) return undefined
+    const source = propagated === undefined ? undefined : parseDisconnectAttribution(propagated)
+    physical.disconnectAttribution = createDisconnectAttribution({
+      ...(source ?? {}),
+      disconnectReason: source?.disconnectReason ?? reason,
+      incidentOrigin: source?.incidentOrigin ?? "reverse_hub",
+      physicalRole: source?.physicalRole ?? physical.role,
+      ...(physical.role === "client" ? {clientConnectionGeneration: physical.generation} : {gatewayConnectionGeneration: physical.generation}),
+      fanOutCount: source?.fanOutCount ?? physical.streams.size,
+      resumeSuccessCount: source?.resumeSuccessCount ?? 0,
+      terminalStreamCount: source?.terminalStreamCount ?? 0
+    })
+    incidentState(physical.disconnectAttribution)
+    return physical.disconnectAttribution
+  }
+
+  /** Preserves the first incident while completing both endpoint generations. */
+  function assignBridgeAttribution(/** @type {Bridge} */ bridge, /** @type {import('./disconnect-attribution.js').DisconnectAttribution} */ attribution) {
+    bridge.disconnectAttribution ??= createDisconnectAttribution({
+      ...currentIncidentAttribution(attribution),
+      clientConnectionGeneration: bridge.client.generation,
+      gatewayConnectionGeneration: bridge.gateway.generation
+    })
+  }
+
+  /** @param {import('./disconnect-attribution.js').DisconnectAttribution} attribution - Incident. @returns {{fanOutCount: number, resumeSuccessCount: number, terminalStreamCount: number}} */
+  function incidentState(attribution) {
+    let state = incidents.get(attribution.disconnectIncidentId)
+    if (state === undefined) {
+      state = {fanOutCount: attribution.fanOutCount, resumeSuccessCount: attribution.resumeSuccessCount, terminalStreamCount: attribution.terminalStreamCount}
+      incidents.set(attribution.disconnectIncidentId, state)
+    } else {
+      state.fanOutCount = Math.max(state.fanOutCount, attribution.fanOutCount)
+      state.resumeSuccessCount = Math.max(state.resumeSuccessCount, attribution.resumeSuccessCount)
+      state.terminalStreamCount = Math.max(state.terminalStreamCount, attribution.terminalStreamCount)
+    }
+    return state
+  }
+
+  /** @param {import('./disconnect-attribution.js').DisconnectAttribution} attribution - Incident. */
+  function currentIncidentAttribution(attribution) {
+    const state = incidentState(attribution)
+    return createDisconnectAttribution({...attribution, ...state})
+  }
+
+  /** @param {import('./disconnect-attribution.js').DisconnectAttribution} attribution - Incident. */
+  function releaseCompletedIncident(attribution) {
+    const state = incidentState(attribution)
+    if (state.resumeSuccessCount + state.terminalStreamCount >= state.fanOutCount) incidents.delete(attribution.disconnectIncidentId)
   }
 
   /** @param {GatewayOwner | ClientOwner} owner - Detached identity. @param {'gateway' | 'client'} role - Owner role. */
   function scheduleOwnerExpiry(owner, role) {
     if (owner.expiryTimer !== undefined) clearTimeout(owner.expiryTimer)
     owner.expiryTimer = setTimeout(() => {
-      owner.expiryTimer = undefined
-      if (owner.physical !== undefined) return
-      for (const bridge of [...owner.streams.values()]) closeBridge(bridge, "RESUME_WINDOW_EXPIRED")
-      if (role === "gateway") {
-        const gatewayOwner = /** @type {GatewayOwner} */ (owner)
-        if (gateways.get(gatewayOwner.gatewayId) === gatewayOwner) gateways.delete(gatewayOwner.gatewayId)
-      } else {
-        const clientOwner = /** @type {ClientOwner} */ (owner)
-        if (clients.get(clientOwner.connectionId) === clientOwner) clients.delete(clientOwner.connectionId)
-      }
-      void tryCompleteResumes()
+      void expireOwner(owner, role)
     }, resumeWindowMs)
     owner.expiryTimer.unref?.()
   }
 
+  /** @param {GatewayOwner | ClientOwner} owner - Detached identity. @param {'gateway' | 'client'} role - Owner role. */
+  async function expireOwner(owner, role) {
+    owner.expiryTimer = undefined
+    if (owner.physical !== undefined) return
+    for (const bridge of [...owner.streams.values()]) await closeBridge(bridge, "RESUME_WINDOW_EXPIRED")
+    if (role === "gateway") {
+      const gatewayOwner = /** @type {GatewayOwner} */ (owner)
+      if (gateways.get(gatewayOwner.gatewayId) === gatewayOwner) gateways.delete(gatewayOwner.gatewayId)
+    } else {
+      const clientOwner = /** @type {ClientOwner} */ (owner)
+      if (clients.get(clientOwner.connectionId) === clientOwner) clients.delete(clientOwner.connectionId)
+    }
+    void tryCompleteResumes()
+  }
+
   /** @param {HubPhysical} physical - Destination. @param {Buffer} bytes - Encoded frame. @param {boolean} [allowClosing] - Allow a final GOAWAY after closing starts. @returns {Promise<boolean>} Delivery result. */
   function enqueueOutput(physical, bytes, allowClosing = false) {
     if (physical.socket.destroyed || (physical.closing && !allowClosing)) return Promise.resolve(false)
@@ -897,6 +1073,8 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
     physical.closing = true
     physical.retainOnDetach = false
     physical.detachCode ??= code
+    physical.firstDisconnectReason ??= disconnectReasonForCode(code)
+    ensurePhysicalAttribution(physical, physical.firstDisconnectReason)
     discardQueuedOutput(physical)
     if (!physical.socket.destroyed) physical.socket.destroy(protocolError(code))
   }
@@ -961,6 +1139,11 @@ function socketKey(socket) {
 /** @param {ReverseGatewayHubOptions} options - Options. */
 function validateOptions(options) {
   if (typeof options !== "object" || options === null || options.tokens === undefined || typeof options.tokens.authenticate !== "function") throw new TypeError("Reverse gateway hub requires a token authenticator")
+  if (options.connectionFingerprint !== undefined) {
+    const {key, keyEpoch} = options.connectionFingerprint
+    if (!(key instanceof Uint8Array) || key.byteLength < 32) throw new TypeError("connection fingerprint key must contain at least 32 bytes")
+    validateFingerprintKeyEpoch(keyEpoch)
+  }
   if (typeof options.authorizeGateway !== "function" || typeof options.authorizeClientGateway !== "function" || typeof options.authorizeOpen !== "function") throw new TypeError("Reverse gateway hub requires gateway, client selection, and open authorization policies")
   if (typeof options.tls !== "object" || options.tls === null || typeof options.tls.certFile !== "string" || typeof options.tls.keyFile !== "string") throw new TypeError("Reverse gateway hub requires TLS certFile and keyFile")
   if (!zstdAvailable()) throw new Error("Reverse gateway hub requires Node.js zstd support")

package/src/reverse/runtime.js

@@ -3,8 +3,8 @@ import {StaticTokenStore} from "../auth/static-token-store.js"
 import {OutboundReverseGateway} from "./gateway.js"
 import {startReverseGatewayHub} from "./hub.js"
 
-/** @param {import('../config/environment.js').RelayEnvironmentConfig} config - Parsed environment. */
-export async function startReverseHubRuntime(config) {
+/** @param {import('../config/environment.js').RelayEnvironmentConfig} config - Parsed environment. @param {import('../logging/structured-logger.js').DiagnosticLogger} [logger] - Optional structured logger. */
+export async function startReverseHubRuntime(config, logger) {
   if (config.mode !== "reverse-hub") throw new Error("Reverse hub runtime requires SOCKETDUCT_MODE=reverse-hub")
   if (config.relayTls === undefined) throw new Error("Reverse hub mode requires SOCKETDUCT_TLS_CERT_FILE and SOCKETDUCT_TLS_KEY_FILE")
   if (config.authTokens.length === 0) throw new Error("SOCKETDUCT_AUTH_TOKENS must contain hub client and gateway credentials")
@@ -54,7 +54,9 @@ export async function startReverseHubRuntime(config) {
     resumeWindowMs: config.resumeWindowMs,
     handshakeTimeoutMs: config.handshakeTimeoutMs,
     maxInputBytesPerConnection: config.maxInputBytesPerConnection,
-    compressionLimits: compressionLimits(config)
+    compressionLimits: compressionLimits(config),
+    ...(logger === undefined ? {} : {logger}),
+    ...(config.connectionFingerprint === undefined ? {} : {connectionFingerprint: config.connectionFingerprint})
   })
   return {hub, close: hub.close}
 }

package/tensorbuzz.yml

@@ -2,7 +2,7 @@ dependency_roots:
   - path: .
 before_install:
   - curl -fsSL https://deb.nodesource.com/setup_24.x | sudo -E bash -
-  - sudo apt-get install -y docker.io nodejs
+  - sudo apt-get install -y docker.io nodejs zstd=1.5.7+dfsg-3
 before_script:
   - node --version
 environment:
@@ -34,4 +34,5 @@ script:
   - npm run typecheck
   - npm run lint
   - npm test
+  - npm run verify:zstd-logs
   - docker build --tag socketduct-ci .

package/test/e2e/logger-shutdown-process.test.js

@@ -0,0 +1,86 @@
+/* eslint-disable jsdoc/escape-inline-tags, jsdoc/require-param, jsdoc/require-returns */
+import assert from "node:assert/strict"
+import {spawn} from "node:child_process"
+import {mkdtemp, rm} from "node:fs/promises"
+import {createServer} from "node:net"
+import {tmpdir} from "node:os"
+import {join} from "node:path"
+import test from "node:test"
+
+import {readCompressedLogDirectory} from "../../src/logging/compressed-log-reader.js"
+
+const TIMEOUT_MS = 8_000
+
+test("SIGTERM durably flushes queued critical lifecycle records before exit", async (t) => {
+  const root = await mkdtemp(join(tmpdir(), "socketduct-logger-shutdown-"))
+  const directory = join(root, "logs")
+  const port = await reservePort()
+  const child = spawn(process.execPath, ["src/index.js"], {
+    cwd: process.cwd(),
+    env: {
+      ...process.env,
+      SOCKETDUCT_MODE: "relay",
+      SOCKETDUCT_RELAY_HOST: "127.0.0.1",
+      SOCKETDUCT_RELAY_PORT: String(port),
+      SOCKETDUCT_AUTH_TOKENS: "shutdown-test:shutdown-test-token",
+      SOCKETDUCT_TARGET_ALLOWLIST: "127.0.0.1:1",
+      SOCKETDUCT_STATUS_INTERVAL_MS: "0",
+      SOCKETDUCT_LOG_DIRECTORY: directory,
+      SOCKETDUCT_LOG_FLUSH_INTERVAL_MS: "60000",
+      SOCKETDUCT_LOG_CRITICAL_FLUSH_INTERVAL_MS: "60000"
+    },
+    stdio: ["ignore", "pipe", "pipe"]
+  })
+  let output = ""
+  child.stdout.on("data", (chunk) => { output += chunk.toString() })
+  child.stderr.on("data", (chunk) => { output += chunk.toString() })
+  t.after(async () => {
+    if (child.exitCode === null) child.kill("SIGKILL")
+    await rm(root, {recursive: true, force: true})
+  })
+
+  await waitUntil(() => output.includes('"event":"relay.started"'), child, () => output)
+  assert.equal(child.kill("SIGTERM"), true)
+  const exit = await withTimeout(new Promise((resolve) => child.once("exit", (code, signal) => resolve({code, signal}))), "process exit")
+  assert.deepEqual(exit, {code: 0, signal: null})
+
+  const result = await readCompressedLogDirectory(directory)
+  assert.equal(result.incomplete, false)
+  assert.deepEqual(result.records.map((record) => record.event), ["process.started", "process.stopped", "log_batch_accepted", "log_frame_committed", "log_closed"])
+})
+
+/** @returns {Promise<number>} A released loopback port. */
+async function reservePort() {
+  const server = createServer()
+  await new Promise((resolve, reject) => {
+    server.once("error", reject)
+    server.listen(0, "127.0.0.1", () => { server.off("error", reject); resolve(undefined) })
+  })
+  const address = server.address()
+  assert(address !== null && typeof address !== "string")
+  await new Promise((resolve) => server.close(() => resolve(undefined)))
+  return address.port
+}
+
+/** @param {() => boolean} predicate - Completion check. @param {import('node:child_process').ChildProcess} child - Child process. @param {() => string} output - Captured output. */
+async function waitUntil(predicate, child, output) {
+  if (predicate()) return
+  await withTimeout(new Promise((resolve, reject) => {
+    const onData = () => { if (predicate()) { cleanup(); resolve(undefined) } }
+    const onExit = (/** @type {number | null} */ code) => { cleanup(); reject(new Error(`process exited ${code}: ${output()}`)) }
+    const cleanup = () => { child.stdout?.off("data", onData); child.stderr?.off("data", onData); child.off("exit", onExit) }
+    child.stdout?.on("data", onData)
+    child.stderr?.on("data", onData)
+    child.once("exit", onExit)
+  }), "process startup")
+}
+
+/** @template T @param {Promise<T>} promise - Work. @param {string} label - Timeout label. @returns {Promise<T>} Result. */
+async function withTimeout(promise, label) {
+  let timer
+  try {
+    return await Promise.race([promise, new Promise((_, reject) => { timer = setTimeout(() => reject(new Error(`Timed out: ${label}`)), TIMEOUT_MS) })])
+  } finally {
+    if (timer !== undefined) clearTimeout(timer)
+  }
+}

package/test/e2e/reverse-gateway-process.test.js

@@ -1,7 +1,7 @@
 /* eslint-disable jsdoc/escape-inline-tags, jsdoc/require-param, jsdoc/require-returns */
 import assert from "node:assert/strict"
 import {spawn} from "node:child_process"
-import {mkdtemp, readFile, readdir, rm} from "node:fs/promises"
+import {mkdtemp, rm} from "node:fs/promises"
 import {createServer} from "node:net"
 import {tmpdir} from "node:os"
 import {join} from "node:path"
@@ -9,10 +9,12 @@ import test from "node:test"
 import {connect as connectTls} from "node:tls"
 
 import {ReverseGatewayConnection} from "../../src/reverse/client.js"
+import {readCompressedLogDirectory} from "../../src/logging/compressed-log-reader.js"
 import {generateTestCertificates} from "../helpers/tls-certificates.js"
 
 const PROCESS_TIMEOUT_MS = 8_000
 const PROCESS_LISTEN_ATTEMPTS = 3
+const TEST_FINGERPRINT_KEY = Buffer.alloc(32, 0x5a).toString("base64url")
 
 /** A child exit observed before its expected startup event. */
 class ProcessStartupExitError extends Error {
@@ -234,8 +236,9 @@ test("spawned reverse hub and outbound gateway expose an allowlisted Docker stre
     SOCKETDUCT_REVERSE_ROUTES: `tensorbuzz:o2:127.0.0.1:${targetPort}`,
     SOCKETDUCT_STATUS_INTERVAL_MS: "0",
     SOCKETDUCT_LOG_DIRECTORY: logDirectory,
-    SOCKETDUCT_LOG_MAX_FILE_BYTES: "4096",
-    SOCKETDUCT_LOG_MAX_FILES: "2"
+    SOCKETDUCT_LOG_SEGMENT_MAX_BYTES: "4096",
+    SOCKETDUCT_CONNECTION_FINGERPRINT_KEY: TEST_FINGERPRINT_KEY,
+    SOCKETDUCT_CONNECTION_FINGERPRINT_KEY_EPOCH: "process-test"
   }, "reverse-hub.started")
   t.after(() => stopProcess(hub))
   const gateway = await startProcess({
@@ -250,8 +253,7 @@ test("spawned reverse hub and outbound gateway expose an allowlisted Docker stre
     SOCKETDUCT_TARGET_ALLOWLIST: `127.0.0.1:${targetPort}`,
     SOCKETDUCT_STATUS_INTERVAL_MS: "0",
     SOCKETDUCT_LOG_DIRECTORY: logDirectory,
-    SOCKETDUCT_LOG_MAX_FILE_BYTES: "4096",
-    SOCKETDUCT_LOG_MAX_FILES: "2"
+    SOCKETDUCT_LOG_SEGMENT_MAX_BYTES: "4096"
   }, "reverse-gateway.started")
   t.after(() => stopProcess(gateway))
 
@@ -270,19 +272,18 @@ test("spawned reverse hub and outbound gateway expose an allowlisted Docker stre
   assert.equal(Buffer.from(/** @type {Uint8Array} */ (await withTimeout(response, "process response"))).toString(), "docker:GET /version")
   assert.equal(client.status().compression, "zstd")
   socket.destroy()
+  client.close()
+  await stopProcess(gateway)
+  await stopProcess(hub)
 
-  assert(hub.pid !== undefined)
-  assert(gateway.pid !== undefined)
-  const files = (await readdir(logDirectory)).sort()
-  assert(files.includes(`reverse-hub-${hub.pid}.jsonl`))
-  assert(files.includes(`reverse-gateway-${gateway.pid}.jsonl`))
-  const hubEntries = (await readFile(join(logDirectory, `reverse-hub-${hub.pid}.jsonl`), "utf8")).trim().split("\n").map((line) => JSON.parse(line))
-  const gatewayContents = await readFile(join(logDirectory, `reverse-gateway-${gateway.pid}.jsonl`), "utf8")
-  const gatewayEntries = gatewayContents.trim().split("\n").map((line) => JSON.parse(line))
+  const logs = await readCompressedLogDirectory(logDirectory)
+  assert.equal(logs.incomplete, false)
+  const hubEntries = logs.records.filter(({role}) => role === "reverse-hub")
+  const gatewayEntries = logs.records.filter(({role}) => role === "reverse-gateway")
   assert(hubEntries.some(({event}) => event === "process.started"))
   assert(gatewayEntries.some(({event}) => event === "process.started"))
   assert(gatewayEntries.some(({event}) => event === "transport.connected"))
-  assert.doesNotMatch(gatewayContents, /process-gateway-token|127\.0\.0\.1|docker-server/)
+  assert.doesNotMatch(JSON.stringify(logs.records), /process-gateway-token|127\.0\.0\.1|docker-server|process_home_o2_gateway/)
 })
 
 test("one route-class client physical connection interleaves streams through different gateway processes", async (t) => {