socketduct 0.1.19 → 0.1.21

package/src/reverse/disconnect-attribution.js

@@ -0,0 +1,191 @@
+/* eslint-disable jsdoc/require-param-type, jsdoc/require-returns, jsdoc/require-returns-description */
+import {createHmac, randomBytes} from "node:crypto"
+
+const INCIDENT_ID = /^[A-Za-z0-9_-]{22}$/
+const FINGERPRINT_KEY_EPOCH = /^[A-Za-z0-9_-]{1,32}$/
+const SAFE_CONTROL_CODE = /^[A-Z][A-Z0-9_]{0,63}$/
+const DISCONNECT_REASONS = new Set([
+  "remote_socket_error",
+  "remote_eof",
+  "peer_goaway",
+  "protocol_violation",
+  "output_backpressure_limit",
+  "heartbeat_timeout",
+  "gateway_replaced",
+  "gateway_shutdown",
+  "hub_shutdown",
+  "local_close",
+  "send_after_transport_loss",
+  "unknown_transport_close"
+])
+const INCIDENT_ORIGINS = new Set(["reverse_hub", "reverse_client", "reverse_gateway"])
+const PHYSICAL_ROLES = new Set(["client", "gateway"])
+const RESERVED_CONTROL_FIELDS = new Set([
+  "disconnectIncidentId",
+  "disconnectReason",
+  "incidentOrigin",
+  "physicalRole",
+  "clientConnectionGeneration",
+  "gatewayConnectionGeneration",
+  "fanOutCount",
+  "resumeSuccessCount",
+  "terminalStreamCount",
+  "connectionFingerprint",
+  "fingerprintKeyEpoch"
+])
+
+/** @typedef {'remote_socket_error' | 'remote_eof' | 'peer_goaway' | 'protocol_violation' | 'output_backpressure_limit' | 'heartbeat_timeout' | 'gateway_replaced' | 'gateway_shutdown' | 'hub_shutdown' | 'local_close' | 'send_after_transport_loss' | 'unknown_transport_close'} DisconnectReason */
+
+/**
+ * @typedef {object} DisconnectAttribution
+ * @property {string} disconnectIncidentId - Random 128-bit incident identity.
+ * @property {DisconnectReason} disconnectReason - First physical cause.
+ * @property {'reverse_hub' | 'reverse_client' | 'reverse_gateway'} incidentOrigin - Component that observed the cause.
+ * @property {'client' | 'gateway'} physicalRole - Physical endpoint role where the cause originated.
+ * @property {number} [clientConnectionGeneration] - Named client physical generation.
+ * @property {number} [gatewayConnectionGeneration] - Named gateway physical generation.
+ * @property {number} fanOutCount - Logical streams affected by the incident.
+ * @property {number} resumeSuccessCount - Logical streams resumed so far.
+ * @property {number} terminalStreamCount - Logical streams made terminal so far.
+ */
+
+/**
+ * Creates one validated disconnect-attribution value.
+ * @param fields
+ */
+export function createDisconnectAttribution(/** @type {Record<string, unknown>} */ fields) {
+  const value = {
+    disconnectIncidentId: fields.disconnectIncidentId ?? randomBytes(16).toString("base64url"),
+    disconnectReason: fields.disconnectReason,
+    incidentOrigin: fields.incidentOrigin,
+    physicalRole: fields.physicalRole,
+    ...(fields.clientConnectionGeneration === undefined ? {} : {clientConnectionGeneration: fields.clientConnectionGeneration}),
+    ...(fields.gatewayConnectionGeneration === undefined ? {} : {gatewayConnectionGeneration: fields.gatewayConnectionGeneration}),
+    fanOutCount: fields.fanOutCount ?? 0,
+    resumeSuccessCount: fields.resumeSuccessCount ?? 0,
+    terminalStreamCount: fields.terminalStreamCount ?? 0
+  }
+  const parsed = parseDisconnectAttribution(value)
+  if (parsed === undefined) throw new TypeError("Invalid disconnect attribution")
+  return parsed
+}
+
+/**
+ * Returns a closed, validated attribution value or undefined.
+ * @param value
+ */
+export function parseDisconnectAttribution(/** @type {unknown} */ value) {
+  if (typeof value !== "object" || value === null || Array.isArray(value)) return undefined
+  const candidate = /** @type {Record<string, unknown>} */ (value)
+  if (typeof candidate.disconnectIncidentId !== "string" || !INCIDENT_ID.test(candidate.disconnectIncidentId)) return undefined
+  if (typeof candidate.disconnectReason !== "string" || !DISCONNECT_REASONS.has(candidate.disconnectReason)) return undefined
+  if (typeof candidate.incidentOrigin !== "string" || !INCIDENT_ORIGINS.has(candidate.incidentOrigin)) return undefined
+  if (typeof candidate.physicalRole !== "string" || !PHYSICAL_ROLES.has(candidate.physicalRole)) return undefined
+  if (!validOptionalGeneration(candidate.clientConnectionGeneration) || !validOptionalGeneration(candidate.gatewayConnectionGeneration)) return undefined
+  if (!validCount(candidate.fanOutCount) || !validCount(candidate.resumeSuccessCount) || !validCount(candidate.terminalStreamCount)) return undefined
+  if (candidate.physicalRole === "client" && candidate.clientConnectionGeneration === undefined) return undefined
+  if (candidate.physicalRole === "gateway" && candidate.gatewayConnectionGeneration === undefined) return undefined
+  return /** @type {DisconnectAttribution} */ ({
+    disconnectIncidentId: candidate.disconnectIncidentId,
+    disconnectReason: /** @type {DisconnectReason} */ (candidate.disconnectReason),
+    incidentOrigin: /** @type {'reverse_hub' | 'reverse_client' | 'reverse_gateway'} */ (candidate.incidentOrigin),
+    physicalRole: /** @type {'client' | 'gateway'} */ (candidate.physicalRole),
+    ...(candidate.clientConnectionGeneration === undefined ? {} : {clientConnectionGeneration: candidate.clientConnectionGeneration}),
+    ...(candidate.gatewayConnectionGeneration === undefined ? {} : {gatewayConnectionGeneration: candidate.gatewayConnectionGeneration}),
+    fanOutCount: candidate.fanOutCount,
+    resumeSuccessCount: candidate.resumeSuccessCount,
+    terminalStreamCount: candidate.terminalStreamCount
+  })
+}
+
+/**
+ * Copies only validated disconnect fields onto a public typed error.
+ * @param error
+ * @param value
+ */
+export function attachDisconnectAttribution(/** @type {Error} */ error, /** @type {unknown} */ value) {
+  const attribution = parseDisconnectAttribution(value)
+  if (attribution !== undefined) Object.assign(error, attribution)
+  return error
+}
+
+/**
+ * Removes fields a peer is never authorized to assert.
+ * @param value
+ */
+export function sanitizePeerControlDetail(/** @type {unknown} */ value) {
+  if (typeof value !== "object" || value === null || Array.isArray(value)) return {detail: {}, reservedFieldsStripped: 0}
+  const candidate = /** @type {Record<string, unknown>} */ (value)
+  let reservedFieldsStripped = 0
+  for (const field of RESERVED_CONTROL_FIELDS) if (Object.hasOwn(candidate, field)) reservedFieldsStripped += 1
+  return {
+    detail: typeof candidate.code === "string" && SAFE_CONTROL_CODE.test(candidate.code) ? {code: candidate.code} : {},
+    reservedFieldsStripped
+  }
+}
+
+/**
+ * Computes the plan-defined hub-only, role-separated 128-bit fingerprint.
+ * @param epochKey
+ * @param physicalRole
+ * @param connectionId
+ */
+export function connectionFingerprint(/** @type {Buffer | Uint8Array} */ epochKey, /** @type {'client' | 'gateway'} */ physicalRole, /** @type {string} */ connectionId) {
+  if (!(epochKey instanceof Uint8Array) || epochKey.byteLength < 32) throw new TypeError("connection fingerprint key must contain at least 32 bytes")
+  if (!PHYSICAL_ROLES.has(physicalRole)) throw new TypeError("physicalRole must be client or gateway")
+  if (typeof connectionId !== "string" || connectionId.length === 0) throw new TypeError("connectionId must be non-empty")
+  return createHmac("sha256", epochKey)
+    .update("socketduct-connection-id:v1")
+    .update(Buffer.from([0]))
+    .update(physicalRole)
+    .update(Buffer.from([0]))
+    .update(connectionId)
+    .digest()
+    .subarray(0, 16)
+    .toString("hex")
+}
+
+/**
+ * Validates the public epoch label used beside keyed fingerprints.
+ * @param value
+ */
+export function validateFingerprintKeyEpoch(/** @type {unknown} */ value) {
+  if (typeof value !== "string" || !FINGERPRINT_KEY_EPOCH.test(value)) throw new TypeError("fingerprintKeyEpoch must be 1-32 URL-safe characters")
+  return value
+}
+
+/** Maps a terminal classification to its first-cause enum when locally initiated. */
+/**
+ * @param code
+ * @param fallback
+ * @returns {DisconnectReason}
+ */
+export function disconnectReasonForCode(/** @type {unknown} */ code, /** @type {DisconnectReason} */ fallback = "unknown_transport_close") {
+  if (code === "OUTPUT_BACKPRESSURE_LIMIT") return "output_backpressure_limit"
+  if (code === "HEARTBEAT_TIMEOUT" || code === "HANDSHAKE_TIMEOUT" || code === "RESUME_WINDOW_EXPIRED" || code === "SOCKETDUCT_AUTH_TIMEOUT") return "heartbeat_timeout"
+  if (code === "GATEWAY_REPLACED" || code === "CONNECTION_REPLACED") return "gateway_replaced"
+  if (code === "GATEWAY_SHUTDOWN") return "gateway_shutdown"
+  if (code === "HUB_SHUTDOWN") return "hub_shutdown"
+  if (typeof code === "string" && (code.includes("PROTOCOL") || code === "STALE_GENERATION" || code === "UNSUPPORTED_FRAME")) return "protocol_violation"
+  return fallback
+}
+
+/**
+ * Returns the safe socket error code for logs without retaining arbitrary text.
+ * @param error
+ */
+export function safeSocketCode(/** @type {unknown} */ error) {
+  if (error === null || typeof error !== "object") return undefined
+  const code = /** @type {{code?: unknown}} */ (error).code
+  return typeof code === "string" && /^(?:E[A-Z0-9_]{1,31}|ERR_(?:TLS|SSL)_[A-Z0-9_]{1,48})$/.test(code) ? code : undefined
+}
+
+/** @param {unknown} value - Candidate generation. */
+function validOptionalGeneration(value) {
+  return value === undefined || (Number.isSafeInteger(value) && /** @type {number} */ (value) >= 0)
+}
+
+/** @param {unknown} value - Candidate count. */
+function validCount(value) {
+  return Number.isSafeInteger(value) && /** @type {number} */ (value) >= 0
+}

package/src/reverse/gateway.js

@@ -25,6 +25,7 @@ import {
   STABLE_IDENTIFIER,
   writeSocket
 } from "./common.js"
+import {attachDisconnectAttribution, createDisconnectAttribution, parseDisconnectAttribution, safeSocketCode} from "./disconnect-attribution.js"
 
 const DEFAULT_STREAM_BUFFER_BYTES = 4 * 1024 * 1024
 const DEFAULT_TOTAL_BUFFER_BYTES = 32 * 1024 * 1024
@@ -124,6 +125,7 @@ export class OutboundReverseGateway extends EventEmitter {
   /** @type {number} */ #spoolBytes = 0
   /** @type {number} */ #spoolCleanupFailures = 0
   /** @type {Set<Promise<void>>} */ #spoolCleanupTasks = new Set()
+  /** @type {import('./disconnect-attribution.js').DisconnectAttribution | undefined} */ #disconnectAttribution
 
   /** @param {OutboundReverseGatewayOptions} options - Identity, hub, TLS and local policy. */
   constructor(options) {
@@ -240,7 +242,16 @@ export class OutboundReverseGateway extends EventEmitter {
     let attemptError
     let authenticated = false
     let registered = false
-    const timer = setTimeout(() => physical.destroy(Object.assign(new Error("Reverse gateway connect/auth/register timeout"), {code: "SOCKETDUCT_AUTH_TIMEOUT"})), this.#options.connectTimeoutMs ?? DEFAULT_CONNECT_TIMEOUT_MS)
+    /** @type {import('./disconnect-attribution.js').DisconnectReason | undefined} */
+    let attemptDisconnectReason
+    /** @type {string | undefined} */
+    let attemptSafeSocketCode
+    /** @type {import('./disconnect-attribution.js').DisconnectAttribution | undefined} */
+    let attemptAttribution
+    const timer = setTimeout(() => {
+      attemptDisconnectReason ??= "heartbeat_timeout"
+      physical.destroy(Object.assign(new Error("Reverse gateway connect/auth/register timeout"), {code: "SOCKETDUCT_AUTH_TIMEOUT"}))
+    }, this.#options.connectTimeoutMs ?? DEFAULT_CONNECT_TIMEOUT_MS)
     timer.unref?.()
     physical.once("secureConnect", () => {
       if (this.#physical !== physical || physical.destroyed) return
@@ -260,13 +271,17 @@ export class OutboundReverseGateway extends EventEmitter {
         for (const frame of decoder.push(chunk)) {
           if (frame.type === V3FrameType.AUTH_ERROR || frame.type === V3FrameType.GOAWAY) {
             const detail = decodeV3Control(frame.payload)
-            throw connectionError(typeof detail.code === "string" ? detail.code : "CONNECTION_REJECTED")
+            if (frame.type === V3FrameType.GOAWAY) attemptDisconnectReason ??= "peer_goaway"
+            const parsedAttribution = parseDisconnectAttribution(detail)
+            if (parsedAttribution?.incidentOrigin === "reverse_hub") attemptAttribution = parsedAttribution
+            throw connectionError(typeof detail.code === "string" ? detail.code : "CONNECTION_REJECTED", attemptAttribution)
           }
           if (!authenticated) {
             if (frame.type !== V3FrameType.AUTH_OK) throw fatalError("AUTH_NEGOTIATION_FAILED")
             const detail = decodeV3Control(frame.payload)
             if (detail.version !== 3 || detail.role !== "gateway" || detail.connectionId !== this.#options.connectionId || detail.generation !== generation || detail.compression !== ZSTD_CODEC || !validPeerWindow(detail.initialWindow)) throw fatalError("AUTH_NEGOTIATION_FAILED")
             authenticated = true
+            this.#disconnectAttribution = undefined
             this.#codec = ZSTD_CODEC
             this.#sendWindowLimit = Math.min(this.#initialWindow(), detail.initialWindow)
             this.#setState("registering")
@@ -298,13 +313,20 @@ export class OutboundReverseGateway extends EventEmitter {
         }
       }).catch((error) => {
         attemptError = asError(error)
+        attemptDisconnectReason ??= "protocol_violation"
         if (/** @type {{fatal?: unknown}} */ (attemptError).fatal === true) this.#fatal = true
         physical.destroy(attemptError)
       }).finally(() => {
         if (!physical.destroyed) physical.resume()
       })
     })
-    physical.once("error", (error) => { attemptError = error; if (isFatalTransportError(error)) this.#fatal = true })
+    physical.once("end", () => { attemptDisconnectReason ??= "remote_eof" })
+    physical.once("error", (error) => {
+      attemptError = error
+      attemptDisconnectReason ??= "remote_socket_error"
+      attemptSafeSocketCode ??= safeSocketCode(error)
+      if (isFatalTransportError(error)) this.#fatal = true
+    })
     physical.once("close", () => {
       clearTimeout(timer)
       void queue.finally(() => {
@@ -315,11 +337,28 @@ export class OutboundReverseGateway extends EventEmitter {
         this.#resumeResults = []
         this.#resumeNextPage = 0
         if (!this.#closing && !this.#fatal) this.#setState("reconnecting")
+        const attribution = attemptAttribution ?? this.#disconnectAttribution ?? createDisconnectAttribution({
+          disconnectReason: attemptDisconnectReason ?? "unknown_transport_close",
+          incidentOrigin: "reverse_gateway",
+          physicalRole: "gateway",
+          gatewayConnectionGeneration: generation,
+          fanOutCount: this.#streams.size,
+          resumeSuccessCount: 0,
+          terminalStreamCount: 0
+        })
+        this.#disconnectAttribution = attribution
         if (!this.#closing) {
+          logDiagnostic(this.#options.logger, "reverse_gateway.physical_detached", {
+            gatewayId: this.#options.gatewayId,
+            ...attribution,
+            ...(attemptSafeSocketCode === undefined ? {} : {safeSocketCode: attemptSafeSocketCode})
+          })
           logDiagnostic(this.#options.logger, "transport.disconnected", {
-            generation,
+            gatewayConnectionGeneration: generation,
             gatewayId: this.#options.gatewayId,
             virtualStreams: this.#streams.size,
+            ...attribution,
+            ...(attemptSafeSocketCode === undefined ? {} : {safeSocketCode: attemptSafeSocketCode}),
             ...errorDiagnostics(attemptError, "TRANSPORT_CLOSED")
           })
         }
@@ -696,7 +735,16 @@ export class OutboundReverseGateway extends EventEmitter {
     const delta = sequence - stream.sendAck
     stream.sendAck = sequence
     let released = 0
-    stream.replay = stream.replay.filter((chunk) => { if (chunk.sequence <= sequence) { released += chunk.data.length; return false } return true })
+    stream.replay = stream.replay.flatMap((chunk) => {
+      const start = chunk.sequence - chunk.data.length
+      if (chunk.sequence <= sequence) { released += chunk.data.length; return [] }
+      if (start < sequence) {
+        const acknowledgedBytes = sequence - start
+        released += acknowledgedBytes
+        return [{...chunk, data: Buffer.from(chunk.data.subarray(acknowledgedBytes))}]
+      }
+      return [chunk]
+    })
     stream.bufferedBytes = Math.max(0, stream.bufferedBytes - released)
     this.#bufferedBytes = Math.max(0, this.#bufferedBytes - released)
     stream.sendWindow = Math.min(stream.sendWindowLimit, stream.sendWindow + delta)
@@ -752,7 +800,18 @@ export class OutboundReverseGateway extends EventEmitter {
   /** @param {number} type - Type. @param {number} streamId - Stream. @param {number} sequence - Sequence. @param {Record<string, unknown>} body - Body. */
   async #sendControl(type, streamId, sequence, body) {
     const physical = this.#physical
-    if (physical === undefined || physical.destroyed) return false
+    if (physical === undefined || physical.destroyed) {
+      this.#disconnectAttribution ??= createDisconnectAttribution({
+        disconnectReason: "send_after_transport_loss",
+        incidentOrigin: "reverse_gateway",
+        physicalRole: "gateway",
+        gatewayConnectionGeneration: this.#generation,
+        fanOutCount: this.#streams.size,
+        resumeSuccessCount: 0,
+        terminalStreamCount: 0
+      })
+      return false
+    }
     return await writeSocket(physical, encodeV3Frame({type, streamId, sequence}, encodeV3Control(body)))
   }
 
@@ -865,9 +924,9 @@ function fatalError(/** @type {string} */ code) {
   return error
 }
 
-function connectionError(/** @type {string} */ code) {
-  if (code === "HUB_SHUTDOWN" || code === "CLIENT_DISCONNECTED" || code === "GATEWAY_DISCONNECTED") return Object.assign(new Error("Reverse gateway hub is temporarily unavailable"), {code, fatal: false})
-  return fatalError(code)
+function connectionError(/** @type {string} */ code, /** @type {unknown} */ attribution = undefined) {
+  if (code === "HUB_SHUTDOWN" || code === "CLIENT_DISCONNECTED" || code === "GATEWAY_DISCONNECTED") return attachDisconnectAttribution(Object.assign(new Error("Reverse gateway hub is temporarily unavailable"), {code, fatal: false}), attribution)
+  return attachDisconnectAttribution(fatalError(code), attribution)
 }
 
 /** @param {unknown} value - Advertised peer receive window. @returns {value is number} Whether valid. */