socketduct 0.1.34 → 0.1.35

package/changelog.d/20261027034500-reverse-open-rejection-diagnostics.md

@@ -0,0 +1 @@
+The reverse hub's periodic status now reports rejected opens broken down by reset code, and clients log each relay reset as a `virtual_stream.reset` diagnostic event, so gateway outages can be told apart from authorization or routing rejections without new instrumentation.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "socketduct",
-  "version": "0.1.34",
+  "version": "0.1.35",
   "description": "Resilient authenticated tunnel/proxy for keeping socket streams alive across flaky links.",
   "type": "module",
   "main": "./src/index.js",

package/src/client/multiplexed-connection.js

@@ -959,6 +959,7 @@ export class MultiplexedSocketductConnection extends EventEmitter {
       const code = typeof detail.code === "string" ? detail.code : "STREAM_RESET"
       stream.remoteClosed = true
       stream.state = "reset"
+      logDiagnostic(this.#options.logger, "virtual_stream.reset", {streamId: stream.id, errorCode: code})
       const lost = code.startsWith("RESUME_") || code === "SESSION_LOST" || code.includes("UNCERTAIN")
       stream.socket._receiveError(lost
         ? new SocketductSessionLostError("Virtual stream could not be resumed", code)

package/src/logging/structured-logger.js

@@ -27,6 +27,7 @@ const COMPRESSION = new Set(["auto", "off", "zstd", "none"])
 const STATES = new Set(["configured", "connecting", "registering", "ready", "reconnecting", "disconnected", "closed", "starting", "degraded", "failed"])
 const SAFE_ERROR_CODES = new Set([
   "AUTH_FAILED",
+  "AMBIGUOUS_ROUTE",
   "AUTH_NEGOTIATION_FAILED",
   "AUTH_OWNERSHIP_MISMATCH",
   "AUTH_REQUIRED",
@@ -51,6 +52,7 @@ const SAFE_ERROR_CODES = new Set([
   "HANDSHAKE_TIMEOUT",
   "HUB_SHUTDOWN",
   "INPUT_LIMIT_EXCEEDED",
+  "INVALID_ROUTE_RESOLUTION",
   "INVALID_STREAM_OPEN",
   "PROTOCOL_NEGOTIATION_FAILED",
   "PROTOCOL_VIOLATION",
@@ -77,6 +79,7 @@ const SAFE_ERROR_CODES = new Set([
   "TARGET_OPEN_TIMEOUT",
   "TARGET_READ_FAILED",
   "TARGET_UNAUTHORIZED",
+  "UNKNOWN_GATEWAY_ID",
   "TLS_ERROR",
   "TRANSPORT_CLOSED",
   "UNKNOWN"
@@ -156,7 +159,7 @@ const NUMBER_FIELDS = new Set([
   "virtualStreams",
   "wireDataBytes"
 ])
-const RUNTIME_STATUS_FIELDS = new Set([...NUMBER_FIELDS, "compression", "physicalState"])
+const RUNTIME_STATUS_FIELDS = new Set([...NUMBER_FIELDS, "compression", "physicalState", "rejectedOpensByCode"])
 const EVENT_FIELDS = new Map([
   ["process.started", new Set(["runtimeMode", "transportSecurity", "compression", "compressionLevel", "frameMaxBytes", "flushIntervalMs", "criticalFlushIntervalMs", "maxSegmentBytes", "retentionDays", "retentionBytes", "maxQueueBytes", "criticalReserveBytes"])],
   ["process.stopped", new Set(["runtimeMode"])],
@@ -174,6 +177,7 @@ const EVENT_FIELDS = new Map([
   ["transport_set.terminal_failure", new Set(["configuredMembers", "closedMembers", "errorCode"])],
   ["virtual_stream.opening", new Set(["streamId", "generation", "gatewayId", "recovered"])],
   ["virtual_stream.opened", new Set(["streamId", "generation", "gatewayId", "recovered"])],
+  ["virtual_stream.reset", new Set(["streamId", "errorCode"])],
   ["virtual_stream.closed", new Set(["streamId", "generation", "gatewayId"])],
   ["virtual_stream.error", new Set(["streamId", "generation", "gatewayId", "routeClass", "errorCode", "disconnectIncidentId", "disconnectReason", "incidentOrigin", "physicalRole", "clientConnectionGeneration", "gatewayConnectionGeneration", "fanOutCount", "resumeSuccessCount", "terminalStreamCount"])],
   ["reverse_gateway.target_opened", new Set(["streamId", "generation", "gatewayId"])],
@@ -249,7 +253,7 @@ export function formatDiagnosticLogLine(options) {
   const base = {timestamp: timestamp.toISOString(), event: options.event, processId, role: options.role, ...envelope}
   let line = formatLogLine(base, safeFields)
   if (Buffer.byteLength(line) > MAX_JSONL_BYTES && safeFields.errorCode !== undefined) {
-    /** @type {Record<string, string | number | boolean>} */
+    /** @type {Record<string, string | number | boolean | Record<string, number>>} */
     const boundedFields = {...safeFields}
     delete boundedFields.errorStack
     if (typeof boundedFields.errorMessage === "string") {
@@ -595,12 +599,12 @@ export function errorDiagnostics(error, fallback = "UNKNOWN") {
 /**
  * @param {string} event - Event name.
  * @param {Record<string, unknown>} fields - Candidate fields.
- * @returns {Record<string, string | number | boolean> | undefined} Allowlisted fields, or undefined for an unknown event.
+ * @returns {Record<string, string | number | boolean | Record<string, number>> | undefined} Allowlisted fields, or undefined for an unknown event.
  */
 function sanitizeFields(event, fields) {
   const allowed = EVENT_FIELDS.get(event)
   if (allowed === undefined || typeof fields !== "object" || fields === null || Array.isArray(fields)) return undefined
-  /** @type {Record<string, string | number | boolean>} */
+  /** @type {Record<string, string | number | boolean | Record<string, number>>} */
   const safe = {}
   for (const field of allowed) {
     const value = fields[field]
@@ -656,6 +660,22 @@ function sanitizeFields(event, fields) {
       const errorCode = normalizeErrorCode(value)
       safe.errorCode = errorCode
       safe.errorClass = classifyError(errorCode)
+    } else if (field === "rejectedOpensByCode" && typeof value === "object" && value !== null && !Array.isArray(value)) {
+      const entries = Object.entries(value)
+      if (entries.length <= 16) {
+        /** @type {Record<string, number>} */
+        const bounded = {}
+        let valid = true
+
+        for (const [code, count] of entries) {
+          if (!SAFE_ERROR_CODES.has(code) || typeof count !== "number" || !Number.isSafeInteger(count) || count < 0) {
+            valid = false
+            break
+          }
+          bounded[code] = count
+        }
+        if (valid) safe.rejectedOpensByCode = bounded
+      }
     }
   }
   const errorDetails = Object.getOwnPropertyDescriptor(fields, ERROR_DETAILS)?.value
@@ -671,7 +691,7 @@ function sanitizeFields(event, fields) {
 
 /**
  * @param {{event: string, processId: number, role: string, timestamp: string}} base - Required JSONL fields.
- * @param {Record<string, string | number | boolean>} fields - Sanitized event fields.
+ * @param {Record<string, string | number | boolean | Record<string, number>>} fields - Sanitized event fields.
  * @returns {string} One JSONL record.
  */
 function formatLogLine(base, fields) {

package/src/reverse/hub.js

@@ -175,6 +175,8 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
   let gatewayRegistrations = 0
   let rejectedRegistrations = 0
   let rejectedOpens = 0
+  /** @type {Map<string, number>} */
+  const rejectedOpensByCode = new Map()
   let rawDataBytes = 0
   let wireDataBytes = 0
   let closing = false
@@ -707,6 +709,12 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
     }
   }
 
+  /** @param {string} code - V3 reset code sent to the rejecting client. */
+  function countRejectedOpen(code) {
+    rejectedOpens += 1
+    rejectedOpensByCode.set(code, (rejectedOpensByCode.get(code) || 0) + 1)
+  }
+
   /** @param {Record<string, unknown>} value - Resume manifest entry. */
   function parseResumeItem(value) {
     const item = /** @type {{streamId?: unknown, identifier?: unknown, clientToGatewaySequence?: unknown, clientToGatewayAck?: unknown, gatewayToClientSequence?: unknown, gatewayToClientAck?: unknown, localEnded?: unknown, remoteEnded?: unknown}} */ (value)
@@ -734,17 +742,17 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
       }
       const candidates = options.resolveOpenGateways?.({principal: client.principal, routeClass: client.routeClass, target})
       if (!Array.isArray(candidates) || candidates.some((candidate) => typeof candidate !== "string" || !GATEWAY_ID.test(candidate))) {
-        rejectedOpens += 1
+        countRejectedOpen("INVALID_ROUTE_RESOLUTION")
         await sendControl(client, V3FrameType.RESET, frame.streamId, 0, {code: "INVALID_ROUTE_RESOLUTION"})
         return
       }
       if (candidates.length === 0) {
-        rejectedOpens += 1
+        countRejectedOpen("TARGET_UNAUTHORIZED")
         await sendControl(client, V3FrameType.RESET, frame.streamId, 0, {code: "TARGET_UNAUTHORIZED"})
         return
       }
       if (candidates.length !== 1 || new Set(candidates).size !== 1) {
-        rejectedOpens += 1
+        countRejectedOpen("AMBIGUOUS_ROUTE")
         await sendControl(client, V3FrameType.RESET, frame.streamId, 0, {code: "AMBIGUOUS_ROUTE"})
         return
       }
@@ -757,15 +765,15 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
       gatewayId = request.gatewayId
     }
     if (options.authorizeOpen({principal: client.principal, gatewayId, target}) !== true) {
-      rejectedOpens += 1
+      countRejectedOpen("TARGET_UNAUTHORIZED")
       await sendControl(client, V3FrameType.RESET, frame.streamId, 0, {code: "TARGET_UNAUTHORIZED"})
       return
     }
     const gatewayOwner = gateways.get(gatewayId)
     const gateway = gatewayOwner?.physical
     if (gatewayOwner === undefined || gateway === undefined || gateway.socket.destroyed || gateway.closing || gateway.resumeRequired || gateway.role !== "gateway" || gateway.gatewayId !== gatewayId || gatewayOwner.gatewayId !== gatewayId || gatewayOwner.userId !== gateway.principal?.userId) {
-      rejectedOpens += 1
       const known = gatewayOwner !== undefined || options.knownGatewayIds?.includes(gatewayId) === true
+      countRejectedOpen(known ? "GATEWAY_UNAVAILABLE" : "UNKNOWN_GATEWAY_ID")
       await sendControl(client, V3FrameType.RESET, frame.streamId, 0, {code: known ? "GATEWAY_UNAVAILABLE" : "UNKNOWN_GATEWAY_ID"})
       return
     }
@@ -1222,6 +1230,7 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
       gatewayRegistrations,
       rejectedRegistrations,
       rejectedOpens,
+      rejectedOpensByCode: Object.fromEntries(rejectedOpensByCode),
       virtualStreams: bridges,
       rawDataBytes,
       wireDataBytes,

package/test/integration/multiplexed-failures.test.js

@@ -80,8 +80,11 @@ describe("multiplexed failures", () => {
   it("authentication and exact-target authorization fail closed", async () => withTestCleanup(async (after) => {
     const relay = await startMultiplexedTcpRelay({tokens: tokens(), authorizeTarget: () => false})
     after(() => relay.close())
+    /** @type {Array<{event: string} & Record<string, unknown>>} */
+    const lifecycle = []
     const unauthorizedConnection = new MultiplexedSocketductConnection({
-      relay: {host: "127.0.0.1", port: relay.port, token: "valid-token"}
+      relay: {host: "127.0.0.1", port: relay.port, token: "valid-token"},
+      logger: {log: (/** @type {string} */ event, /** @type {Record<string, unknown>} */ fields = {}) => lifecycle.push({event, ...fields})}
     })
     after(() => unauthorizedConnection.close())
     const denied = unauthorizedConnection.createConnection({host: "127.0.0.1", port: 9})
@@ -89,6 +92,7 @@ describe("multiplexed failures", () => {
     expect(deniedError instanceof SocketductStreamError).toBeTruthy()
     expect(deniedError.code).toBe("TARGET_UNAUTHORIZED")
     expect(relay.status().activeTargets).toBe(0)
+    expect({value: lifecycle.find((entry) => entry.event === "virtual_stream.reset")?.errorCode, context: "the client logs the relay reset code"}).toMatchObject({value: "TARGET_UNAUTHORIZED"})
 
     const invalidConnection = new MultiplexedSocketductConnection({
       relay: {host: "127.0.0.1", port: relay.port, token: "wrong-token"},

package/test/integration/reverse-route-class.test.js

@@ -533,6 +533,7 @@ describe("reverse gateway route-class", () => {
     const reset = await nextError(unavailableStream)
     expect(reset.code).toBe("GATEWAY_UNAVAILABLE")
     unavailableStream.destroy()
+    expect({value: hub.status().rejectedOpensByCode?.GATEWAY_UNAVAILABLE, context: "the hub status counts the rejected open by its reset code"}).toMatchObject({value: 1})
 
     // The route class is healthy, so the reset must not take it out of service.
     // A member marked unavailable this way is never selected again, which would

package/test/unit/logging/structured-logger.test.js

@@ -468,6 +468,17 @@ describe("structured logger", {timeoutMs: 15000}, () => {
     }
   })
 
+  it("sanitizes the rejected-opens-by-code histogram", () => {
+    /** @type {string[]} */
+    const lines = []
+    const logger = createStructuredLogger({role: "relay", writeLine: (line) => lines.push(line)})
+    logger.log("runtime.status", {rejectedOpensByCode: {GATEWAY_UNAVAILABLE: 3, UNKNOWN_GATEWAY_ID: 1}})
+    expect(JSON.parse(lines[0] ?? "{}").rejectedOpensByCode).toEqual({GATEWAY_UNAVAILABLE: 3, UNKNOWN_GATEWAY_ID: 1})
+
+    logger.log("runtime.status", {rejectedOpensByCode: {GATEWAY_UNAVAILABLE: 3, PRIVATE_TOKEN: 2}})
+    expect(JSON.parse(lines[1] ?? "{}").rejectedOpensByCode).toBeUndefined()
+  })
+
   it("oversized sanitized records fall back to a bounded envelope", () => {
     /** @type {string[]} */
     const lines = []