socketduct 0.1.19 → 0.1.21

package/src/logging/structured-logger.js

@@ -14,6 +14,13 @@ const MAX_ERROR_STACK_BYTES = 512
 const ERROR_DETAILS = Symbol("errorDetails")
 const ROLE = /^[a-z][a-z0-9-]{0,63}$/
 const GATEWAY_ID = /^[A-Za-z0-9_-]+$/
+const ROUTE_CLASS = /^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$/
+const DISCONNECT_INCIDENT_ID = /^[A-Za-z0-9_-]{22}$/
+const CONNECTION_FINGERPRINT = /^[a-f0-9]{32}$/
+const FINGERPRINT_KEY_EPOCH = /^[A-Za-z0-9_-]{1,32}$/
+const DISCONNECT_REASONS = new Set(["remote_socket_error", "remote_eof", "peer_goaway", "protocol_violation", "output_backpressure_limit", "heartbeat_timeout", "gateway_replaced", "gateway_shutdown", "hub_shutdown", "local_close", "send_after_transport_loss", "unknown_transport_close"])
+const INCIDENT_ORIGINS = new Set(["reverse_hub", "reverse_client", "reverse_gateway"])
+const PHYSICAL_ROLES = new Set(["client", "gateway"])
 const RUNTIME_MODES = new Set(["summary", "relay", "reverse-hub", "reverse-gateway"])
 const TRANSPORT_SECURITY = new Set(["plaintext", "TLS"])
 const COMPRESSION = new Set(["auto", "off", "zstd", "none"])
@@ -85,22 +92,35 @@ const NUMBER_FIELDS = new Set([
   "bufferLimitBytes",
   "clientPhysicalConnections",
   "closedMembers",
+  "compressionLevel",
   "compressionBytesSaved",
+  "coalescedDrops",
   "configuredMembers",
   "connectingMembers",
   "connectionGeneration",
+  "clientConnectionGeneration",
   "connectionIdentities",
   "controlQueuedBytes",
+  "criticalDrops",
+  "criticalFlushIntervalMs",
+  "criticalReserveBytes",
   "dataQueuedBytes",
   "degradedMembers",
   "delayMs",
   "detachedSessions",
   "gatewayPhysicalConnections",
+  "gatewayConnectionGeneration",
   "gatewayRegistrations",
   "generation",
+  "fanOutCount",
+  "flushIntervalMs",
+  "frameMaxBytes",
   "healthyGateways",
   "maxPendingOpens",
+  "maxQueueBytes",
+  "maxSegmentBytes",
   "minimumReadyMembers",
+  "noncriticalDrops",
   "openFailures",
   "pendingOpens",
   "physicalAccepts",
@@ -115,7 +135,11 @@ const NUMBER_FIELDS = new Set([
   "rejectedOpens",
   "rejectedRegistrations",
   "retainedClientIdentities",
+  "retentionBytes",
+  "retentionDays",
   "resets",
+  "reservedFieldsStripped",
+  "resumeSuccessCount",
   "resumeFailures",
   "resumeSuccesses",
   "sessions",
@@ -127,17 +151,19 @@ const NUMBER_FIELDS = new Set([
   "targetOpenFailures",
   "targetOpens",
   "targetRetryTimers",
+  "terminalStreamCount",
   "virtualOpens",
   "virtualStreams",
   "wireDataBytes"
 ])
 const RUNTIME_STATUS_FIELDS = new Set([...NUMBER_FIELDS, "compression", "physicalState"])
 const EVENT_FIELDS = new Map([
-  ["process.started", new Set(["runtimeMode", "transportSecurity", "compression"])],
+  ["process.started", new Set(["runtimeMode", "transportSecurity", "compression", "compressionLevel", "frameMaxBytes", "flushIntervalMs", "criticalFlushIntervalMs", "maxSegmentBytes", "retentionDays", "retentionBytes", "maxQueueBytes", "criticalReserveBytes"])],
+  ["process.stopped", new Set(["runtimeMode"])],
   ["runtime.status", RUNTIME_STATUS_FIELDS],
   ["transport.connecting", new Set(["generation", "attempt", "gatewayId", "reconnect"])],
   ["transport.connected", new Set(["generation", "attempt", "gatewayId", "physicalConnects", "reconnect"])],
-  ["transport.disconnected", new Set(["generation", "gatewayId", "virtualStreams", "errorCode"])],
+  ["transport.disconnected", new Set(["gatewayId", "routeClass", "virtualStreams", "errorCode", "disconnectIncidentId", "disconnectReason", "incidentOrigin", "physicalRole", "clientConnectionGeneration", "gatewayConnectionGeneration", "fanOutCount", "resumeSuccessCount", "terminalStreamCount", "safeSocketCode"])],
   ["transport.reconnect_scheduled", new Set(["generation", "attempt", "delayMs", "gatewayId", "errorCode"])],
   ["transport.reconnect_succeeded", new Set(["generation", "attempt", "gatewayId", "physicalConnects"])],
   ["transport.failed", new Set(["generation", "attempt", "gatewayId", "errorCode"])],
@@ -149,10 +175,28 @@ const EVENT_FIELDS = new Map([
   ["virtual_stream.opening", new Set(["streamId", "generation", "gatewayId", "recovered"])],
   ["virtual_stream.opened", new Set(["streamId", "generation", "gatewayId", "recovered"])],
   ["virtual_stream.closed", new Set(["streamId", "generation", "gatewayId"])],
-  ["virtual_stream.error", new Set(["streamId", "generation", "gatewayId", "errorCode"])],
+  ["virtual_stream.error", new Set(["streamId", "generation", "gatewayId", "routeClass", "errorCode", "disconnectIncidentId", "disconnectReason", "incidentOrigin", "physicalRole", "clientConnectionGeneration", "gatewayConnectionGeneration", "fanOutCount", "resumeSuccessCount", "terminalStreamCount"])],
   ["reverse_gateway.target_opened", new Set(["streamId", "generation", "gatewayId"])],
   ["reverse_gateway.target_open_failed", new Set(["streamId", "generation", "gatewayId", "errorCode"])],
-  ["reverse_gateway.target_closed", new Set(["streamId", "generation", "gatewayId", "errorCode"])]
+  ["reverse_gateway.target_closed", new Set(["streamId", "generation", "gatewayId", "errorCode"])],
+  ["reverse_hub.physical_detached", new Set(["disconnectIncidentId", "disconnectReason", "incidentOrigin", "physicalRole", "clientConnectionGeneration", "gatewayConnectionGeneration", "fanOutCount", "resumeSuccessCount", "terminalStreamCount", "connectionFingerprint", "fingerprintKeyEpoch", "safeSocketCode"])],
+  ["reverse_hub.streams_interrupted", new Set(["disconnectIncidentId", "disconnectReason", "incidentOrigin", "physicalRole", "clientConnectionGeneration", "gatewayConnectionGeneration", "fanOutCount", "resumeSuccessCount", "terminalStreamCount", "gatewayId", "routeClass"])],
+  ["reverse_hub.streams_resumed", new Set(["disconnectIncidentId", "disconnectReason", "incidentOrigin", "physicalRole", "clientConnectionGeneration", "gatewayConnectionGeneration", "fanOutCount", "resumeSuccessCount", "terminalStreamCount", "gatewayId", "routeClass"])],
+  ["reverse_hub.streams_reset", new Set(["disconnectIncidentId", "disconnectReason", "incidentOrigin", "physicalRole", "clientConnectionGeneration", "gatewayConnectionGeneration", "fanOutCount", "resumeSuccessCount", "terminalStreamCount", "gatewayId", "routeClass", "errorCode"])],
+  ["reverse_hub.reserved_control_field_stripped", new Set(["physicalRole", "clientConnectionGeneration", "gatewayConnectionGeneration", "reservedFieldsStripped"])],
+  ["reverse_client.physical_detached", new Set(["gatewayId", "routeClass", "disconnectIncidentId", "disconnectReason", "incidentOrigin", "physicalRole", "clientConnectionGeneration", "gatewayConnectionGeneration", "fanOutCount", "resumeSuccessCount", "terminalStreamCount", "safeSocketCode"])],
+  ["reverse_client.send_after_transport_loss", new Set(["gatewayId", "routeClass", "disconnectIncidentId", "disconnectReason", "incidentOrigin", "physicalRole", "clientConnectionGeneration", "gatewayConnectionGeneration", "fanOutCount", "resumeSuccessCount", "terminalStreamCount"])],
+  ["reverse_gateway.physical_detached", new Set(["gatewayId", "disconnectIncidentId", "disconnectReason", "incidentOrigin", "physicalRole", "clientConnectionGeneration", "gatewayConnectionGeneration", "fanOutCount", "resumeSuccessCount", "terminalStreamCount", "safeSocketCode"])],
+  ["log_batch_accepted", new Set(["queuedBytes"])],
+  ["log_frame_committed", new Set(["queuedBytes"])],
+  ["log_orphan_recovered", new Set([])],
+  ["log_worker_restarted", new Set(["attempt"])],
+  ["log_queue_pressure", new Set(["queuedBytes", "criticalDrops", "noncriticalDrops", "coalescedDrops"])],
+  ["log_records_dropped", new Set(["criticalDrops", "noncriticalDrops", "coalescedDrops"])],
+  ["log_flush_failed", new Set(["errorCode"])],
+  ["log_sink_failed", new Set(["errorCode"])],
+  ["log_sink_recovered", new Set(["attempt"])],
+  ["log_closed", new Set(["criticalDrops", "noncriticalDrops", "coalescedDrops"])]
 ])
 const FAILED_LOGGERS = new WeakSet()
 
@@ -185,6 +229,60 @@ export function createStructuredLogger(options) {
   return createStructuredLoggerWithFailureCode(options, "LOGGER_WRITER_ERROR")
 }
 
+/**
+ * Produces one bounded, allowlisted JSONL record for a trusted logger envelope.
+ * Unknown events return undefined. Payload-like caller fields never survive the
+ * event allowlist.
+ * @param {{role: string, processId?: number, now?: () => Date, event: string, fields?: Record<string, unknown>, envelope?: {writerEpoch: string, eventSequence: string}}} options - Record inputs.
+ * @returns {string | undefined} Complete newline-terminated JSONL record.
+ */
+export function formatDiagnosticLogLine(options) {
+  validateRole(options?.role)
+  const processId = validateProcessId(options.processId ?? process.pid)
+  const now = options.now ?? (() => new Date())
+  if (typeof now !== "function") throw new TypeError("now must be a function")
+  const safeFields = sanitizeFields(options.event, options.fields ?? {})
+  if (safeFields === undefined) return undefined
+  const timestamp = now()
+  if (!(timestamp instanceof Date) || !Number.isFinite(timestamp.getTime())) throw new TypeError("Logger clock returned an invalid date")
+  const envelope = options.envelope === undefined ? {} : validateCompressedEnvelope(options.envelope)
+  const base = {timestamp: timestamp.toISOString(), event: options.event, processId, role: options.role, ...envelope}
+  let line = formatLogLine(base, safeFields)
+  if (Buffer.byteLength(line) > MAX_JSONL_BYTES && safeFields.errorCode !== undefined) {
+    /** @type {Record<string, string | number | boolean>} */
+    const boundedFields = {...safeFields}
+    delete boundedFields.errorStack
+    if (typeof boundedFields.errorMessage === "string") {
+      const emptyMessageLine = formatLogLine(base, {...boundedFields, errorMessage: ""})
+      const availableBytes = Math.max(0, MAX_JSONL_BYTES - Buffer.byteLength(emptyMessageLine))
+      boundedFields.errorMessage = truncateDiagnosticText(boundedFields.errorMessage, availableBytes)
+    }
+    line = formatLogLine(base, boundedFields)
+    if (Buffer.byteLength(line) > MAX_JSONL_BYTES) {
+      delete boundedFields.errorMessage
+      line = formatLogLine(base, boundedFields)
+    }
+    if (Buffer.byteLength(line) > MAX_JSONL_BYTES) {
+      delete boundedFields.errorName
+      line = formatLogLine(base, boundedFields)
+    }
+  }
+  if (Buffer.byteLength(line) > MAX_JSONL_BYTES) line = formatLogLine(base, {})
+  if (Buffer.byteLength(line) > MAX_JSONL_BYTES) throw new Error("Trusted logger envelope exceeds the JSONL record bound")
+  return line
+}
+
+/**
+ * @param {Record<string, unknown>} envelope - Producer-owned durable identity.
+ * @returns {{writerEpoch: string, eventSequence: string}} Validated fields.
+ */
+function validateCompressedEnvelope(envelope) {
+  if (Object.keys(envelope).length !== 2 || !Object.hasOwn(envelope, "writerEpoch") || !Object.hasOwn(envelope, "eventSequence")) throw new TypeError("compressed logger envelope must contain only writerEpoch and eventSequence")
+  if (typeof envelope.writerEpoch !== "string" || !/^[A-Za-z0-9_-]{22}$/.test(envelope.writerEpoch) || Buffer.from(envelope.writerEpoch, "base64url").toString("base64url") !== envelope.writerEpoch) throw new TypeError("compressed logger writerEpoch must be canonical 128-bit base64url")
+  if (typeof envelope.eventSequence !== "string" || !/^[1-9][0-9]*$/.test(envelope.eventSequence) || BigInt(envelope.eventSequence) > 0xffff_ffff_ffff_ffffn) throw new TypeError("compressed logger eventSequence must be an unsigned 64-bit decimal string")
+  return {writerEpoch: envelope.writerEpoch, eventSequence: envelope.eventSequence}
+}
+
 /**
  * @param {StructuredLoggerOptions} options - Logger identity and writer.
  * @param {'LOGGER_IO_ERROR' | 'LOGGER_WRITER_ERROR'} failureCode - Fixed failure classification.
@@ -209,32 +307,8 @@ function createStructuredLoggerWithFailureCode(options, failureCode) {
   return {
     log(event, fields = {}) {
       try {
-        const safeFields = sanitizeFields(event, fields)
-        if (safeFields === undefined) return
-        const timestamp = now()
-        if (!(timestamp instanceof Date) || !Number.isFinite(timestamp.getTime())) throw new TypeError("Logger clock returned an invalid date")
-        const base = {timestamp: timestamp.toISOString(), event, processId, role}
-        let line = formatLogLine(base, safeFields)
-        if (Buffer.byteLength(line) > MAX_JSONL_BYTES && safeFields.errorCode !== undefined) {
-          /** @type {Record<string, string | number | boolean>} */
-          const boundedFields = {...safeFields}
-          delete boundedFields.errorStack
-          if (typeof boundedFields.errorMessage === "string") {
-            const emptyMessageLine = formatLogLine(base, {...boundedFields, errorMessage: ""})
-            const availableBytes = Math.max(0, MAX_JSONL_BYTES - Buffer.byteLength(emptyMessageLine))
-            boundedFields.errorMessage = truncateDiagnosticText(boundedFields.errorMessage, availableBytes)
-          }
-          line = formatLogLine(base, boundedFields)
-          if (Buffer.byteLength(line) > MAX_JSONL_BYTES) {
-            delete boundedFields.errorMessage
-            line = formatLogLine(base, boundedFields)
-          }
-          if (Buffer.byteLength(line) > MAX_JSONL_BYTES) {
-            delete boundedFields.errorName
-            line = formatLogLine(base, boundedFields)
-          }
-        }
-        if (Buffer.byteLength(line) > MAX_JSONL_BYTES) line = formatLogLine(base, {})
+        const line = formatDiagnosticLogLine({role, processId, now, event, fields})
+        if (line === undefined) return
         observeRejection(writeLine(line), reportOnce)
       } catch {
         reportOnce()
@@ -541,6 +615,38 @@ function sanitizeFields(event, fields) {
       }
       continue
     }
+    if (field === "routeClass") {
+      if (typeof value === "string" && ROUTE_CLASS.test(value)) safe.routeClass = value
+      continue
+    }
+    if (field === "disconnectIncidentId") {
+      if (typeof value === "string" && DISCONNECT_INCIDENT_ID.test(value)) safe.disconnectIncidentId = value
+      continue
+    }
+    if (field === "disconnectReason") {
+      if (typeof value === "string" && DISCONNECT_REASONS.has(value)) safe.disconnectReason = value
+      continue
+    }
+    if (field === "incidentOrigin") {
+      if (typeof value === "string" && INCIDENT_ORIGINS.has(value)) safe.incidentOrigin = value
+      continue
+    }
+    if (field === "physicalRole") {
+      if (typeof value === "string" && PHYSICAL_ROLES.has(value)) safe.physicalRole = value
+      continue
+    }
+    if (field === "connectionFingerprint") {
+      if (typeof value === "string" && CONNECTION_FINGERPRINT.test(value)) safe.connectionFingerprint = value
+      continue
+    }
+    if (field === "fingerprintKeyEpoch") {
+      if (typeof value === "string" && FINGERPRINT_KEY_EPOCH.test(value)) safe.fingerprintKeyEpoch = value
+      continue
+    }
+    if (field === "safeSocketCode") {
+      if (typeof value === "string" && /^(?:E[A-Z0-9_]{1,31}|ERR_(?:TLS|SSL)_[A-Z0-9_]{1,48})$/.test(value)) safe.safeSocketCode = value
+      continue
+    }
     if (field === "runtimeMode" && typeof value === "string" && RUNTIME_MODES.has(value)) safe.runtimeMode = value
     else if (field === "transportSecurity" && typeof value === "string" && TRANSPORT_SECURITY.has(value)) safe.transportSecurity = value
     else if (field === "compression" && typeof value === "string" && COMPRESSION.has(value)) safe.compression = value

package/src/logging/zstd-frame-codec.js

@@ -0,0 +1,73 @@
+/* eslint-disable jsdoc/require-param-type, jsdoc/require-returns */
+import {promisify} from "node:util"
+import {constants, zstdCompress, zstdDecompress} from "node:zlib"
+
+export const DEFAULT_LOG_COMPRESSION_LEVEL = 19
+
+const compress = promisify(zstdCompress)
+const decompress = promisify(zstdDecompress)
+
+/**
+ * Compresses one complete JSONL batch as one standard checksummed zstd frame.
+ * @param input
+ * @param level
+ */
+export async function compressLogFrame(/** @type {Buffer | Uint8Array} */ input, /** @type {number} */ level = DEFAULT_LOG_COMPRESSION_LEVEL) {
+  if (!(input instanceof Uint8Array) || input.byteLength === 0) throw new TypeError("log frame input must be non-empty bytes")
+  if (!Number.isInteger(level) || level < -131_072 || level > 22) throw new RangeError("log compression level is outside the zstd range")
+  const frame = await compress(input, {
+    params: {
+      [constants.ZSTD_c_compressionLevel]: level,
+      [constants.ZSTD_c_checksumFlag]: 1,
+      [constants.ZSTD_c_contentSizeFlag]: 1
+    }
+  })
+  const encoded = Buffer.from(frame)
+  inspectZstdFrame(encoded)
+  return encoded
+}
+
+/**
+ * Decodes one independently indexed zstd frame.
+ * @param frame
+ */
+export async function decompressLogFrame(/** @type {Buffer | Uint8Array} */ frame) {
+  if (!(frame instanceof Uint8Array) || frame.byteLength === 0) throw new TypeError("zstd frame must be non-empty bytes")
+  inspectZstdFrame(frame)
+  return Buffer.from(await decompress(frame))
+}
+
+/**
+ * Requires exactly one complete standard zstd frame with a checksum trailer.
+ * @param value
+ */
+export function inspectZstdFrame(/** @type {Buffer | Uint8Array} */ value) {
+  const frame = Buffer.from(value)
+  if (frame.length < 9 || frame.readUInt32LE(0) !== 0xfd2f_b528) throw new Error("Invalid zstd frame magic")
+  const descriptor = /** @type {number} */ (frame[4])
+  const contentSizeFlag = descriptor >>> 6
+  const singleSegment = (descriptor & 0x20) !== 0
+  const checksum = (descriptor & 0x04) !== 0
+  const dictionaryFlag = descriptor & 0x03
+  if ((descriptor & 0x08) !== 0) throw new Error("Reserved zstd frame descriptor bit is set")
+  if (!checksum) throw new Error("Zstd log frame checksum flag is required")
+  const dictionaryBytes = /** @type {number} */ ([0, 1, 2, 4][dictionaryFlag])
+  const contentSizeBytes = contentSizeFlag === 0 ? singleSegment ? 1 : 0 : contentSizeFlag === 1 ? 2 : contentSizeFlag === 2 ? 4 : 8
+  let offset = 5 + (singleSegment ? 0 : 1) + dictionaryBytes + contentSizeBytes
+  if (offset > frame.length) throw new Error("Truncated zstd frame header")
+  let lastBlock = false
+  while (!lastBlock) {
+    if (offset + 3 > frame.length) throw new Error("Truncated zstd block header")
+    const header = /** @type {number} */ (frame[offset]) | (/** @type {number} */ (frame[offset + 1]) << 8) | (/** @type {number} */ (frame[offset + 2]) << 16)
+    offset += 3
+    lastBlock = (header & 1) !== 0
+    const blockType = (header >>> 1) & 0x03
+    const blockSize = header >>> 3
+    if (blockType === 3) throw new Error("Reserved zstd block type")
+    const encodedBytes = blockType === 1 ? 1 : blockSize
+    if (offset + encodedBytes > frame.length) throw new Error("Truncated zstd block")
+    offset += encodedBytes
+  }
+  if (offset + 4 !== frame.length) throw new Error(offset + 4 > frame.length ? "Truncated zstd checksum" : "Zstd frame contains trailing bytes")
+  return {bytes: frame.length, checksum: true}
+}