socketduct 0.1.28 → 0.1.29

package/changelog.d/20261025123000-route-class-down-registration.md

@@ -0,0 +1 @@
+A route class the hub reports down is never selected while it stays down: the member keeps re-reaching the hub on a bounded backoff and returns to service automatically once a gateway registers again, and `status()` now exposes the down state and the probe attempts.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "socketduct",
-  "version": "0.1.28",
+  "version": "0.1.29",
   "description": "Resilient authenticated tunnel/proxy for keeping socket streams alive across flaky links.",
   "type": "module",
   "main": "./src/index.js",

package/src/client/multiplexed-transport-set.js

@@ -493,13 +493,14 @@ export class MultiplexedSocketductTransportSet extends EventEmitter {
     return this
   }
 
-  /** @returns {{state: string, configuredMembers: number, connectingMembers: number, readyMembers: number, degradedMembers: number, closedMembers: number, minimumReadyMembers: number, pendingOpens: number, maxPendingOpens: number, virtualOpens: number, physicalConnects: number, reconnectAttempts: number, budgets: {dataBytes: number, spoolBytes: number, queuedBytes: number, maxDataBytes: number, maxSpoolBytes: number, maxQueuedBytes: number}, members: Array<{gatewayId?: string, routeClass?: string, state: string, assignments: number, physicalConnects: number, reconnectAttempts: number, reconnectSuccesses: number, virtualStreams: number, queuedBytes: number, spoolBytes: number, spoolCleanupFailures: number, terminalErrorCode: string | null}>}} Privacy-safe aggregate and per-member status. */
+  /** @returns {{state: string, configuredMembers: number, connectingMembers: number, readyMembers: number, degradedMembers: number, closedMembers: number, minimumReadyMembers: number, pendingOpens: number, maxPendingOpens: number, virtualOpens: number, physicalConnects: number, reconnectAttempts: number, budgets: {dataBytes: number, spoolBytes: number, queuedBytes: number, maxDataBytes: number, maxSpoolBytes: number, maxQueuedBytes: number}, members: Array<{gatewayId?: string, routeClass?: string, state: string, gatewayUnavailable: boolean, assignments: number, physicalConnects: number, reconnectAttempts: number, reconnectSuccesses: number, virtualStreams: number, queuedBytes: number, spoolBytes: number, spoolCleanupFailures: number, terminalErrorCode: string | null}>}} Privacy-safe aggregate and per-member status. */
   status() {
     const members = this.#members.map((member) => {
       const status = member.connection.status()
       return {
         ...memberAllocation(member),
         state: status.physicalState,
+        gatewayUnavailable: status.gatewayUnavailable === true,
         assignments: member.assignments,
         physicalConnects: status.physicalConnects,
         reconnectAttempts: status.reconnectAttempts,
@@ -571,33 +572,25 @@ export class MultiplexedSocketductTransportSet extends EventEmitter {
   }
 
   /**
-   * Selects the next ready member, preferring members whose gateway the hub has
-   * not recently reported unavailable. When every ready member carries a
-   * gateway-unavailable mark, any ready member may still be selected so a stale
-   * mark can never wedge the set.
+   * Selects the next ready member whose route class the hub has registered up.
+   * A member the hub registered down is never selected: nothing is sent to a
+   * route class without a reachable gateway. When every member is registered
+   * down the open stays pending here until a member reports availability again
+   * (the member keeps probing the hub, and its state change re-drains this set).
    * @returns {{member: TransportMember, previousCursor: number} | undefined}
    */
   #selectReadyMember() {
     if (this.#closed) throw closedError()
     const previousCursor = this.#cursor
-    /** @type {{member: TransportMember, index: number} | undefined} */
-    let fallback
     for (let offset = 0; offset < this.#members.length; offset += 1) {
       const index = (this.#cursor + offset) % this.#members.length
       const member = /** @type {TransportMember} */ (this.#members[index])
       const status = member.connection.status()
       if (status.physicalState !== "ready") continue
-      if (status.gatewayUnavailable === true) {
-        fallback ??= {member, index}
-        continue
-      }
+      if (status.gatewayUnavailable === true) continue
       this.#cursor = (index + 1) % this.#members.length
       return {member, previousCursor}
     }
-    if (fallback !== undefined) {
-      this.#cursor = (fallback.index + 1) % this.#members.length
-      return {member: fallback.member, previousCursor}
-    }
     return undefined
   }
 

package/src/reverse/client.js

@@ -32,6 +32,8 @@ import {attachDisconnectAttribution, createDisconnectAttribution, parseDisconnec
 const DEFAULT_STREAM_BUFFER_BYTES = 4 * 1024 * 1024
 const DEFAULT_CONNECTION_BUFFER_BYTES = 32 * 1024 * 1024
 const MANIFEST_PAGE_STREAMS = 64
+const DEFAULT_GATEWAY_PROBE_DELAY_MS = 1_000
+const DEFAULT_MAX_GATEWAY_PROBE_DELAY_MS = 30_000
 
 /**
  * @typedef {object} ReverseGatewayConnectionOptions
@@ -50,6 +52,8 @@ const MANIFEST_PAGE_STREAMS = 64
  * @property {number} [connectTimeoutMs] - TLS/auth deadline.
  * @property {number} [reconnectDelayMs] - Initial reconnect delay.
  * @property {number} [maxReconnectDelayMs] - Maximum reconnect delay.
+ * @property {number} [gatewayProbeDelayMs] - Initial re-reach delay while the hub reports no selectable gateway.
+ * @property {number} [maxGatewayProbeDelayMs] - Maximum re-reach delay while the hub reports no selectable gateway.
  * @property {number} [initialGeneration] - Monotonic seed for controlled replacement; defaults to wall-clock microseconds.
  * @property {boolean} [continuousReconnect] - Keep the member warm after loss, default true.
  * @property {import('../protocol/compression.js').CompressionLimits} [compressionLimits] - zstd safety bounds.
@@ -168,7 +172,10 @@ export class ReverseGatewayConnection extends EventEmitter {
   /** @type {number} */ #spoolBytes = 0
   /** @type {number} */ #spoolCleanupFailures = 0
   /** @type {Set<Promise<void>>} */ #spoolCleanupTasks = new Set()
-  /** @type {number | undefined} - Wall-clock time of the last hub-reported gateway unavailability. */ #gatewayUnavailableSince
+  /** @type {number | undefined} - Wall-clock time the hub first reported this member's route class unselectable; undefined means registered available. */ #gatewayDownSince
+  /** @type {ReturnType<typeof setTimeout> | undefined} */ #gatewayProbeTimer
+  /** @type {number} */ #gatewayProbeDelay
+  /** @type {number} */ #gatewayProbeAttempts = 0
   /** @type {import('./disconnect-attribution.js').DisconnectAttribution | undefined} */ #disconnectAttribution
 
   /** @param {ReverseGatewayConnectionOptions} options - Hub route and safety policy. */
@@ -184,6 +191,7 @@ export class ReverseGatewayConnection extends EventEmitter {
     })
     this.#generation = options.initialGeneration ?? Date.now() * 1000
     this.#reconnectDelay = options.reconnectDelayMs ?? 250
+    this.#gatewayProbeDelay = options.gatewayProbeDelayMs ?? DEFAULT_GATEWAY_PROBE_DELAY_MS
     this.#sendWindowLimit = this.#initialWindow()
     this.createConnection = this.createConnection.bind(this)
   }
@@ -360,30 +368,92 @@ export class ReverseGatewayConnection extends EventEmitter {
    * @returns {void}
    */
   #markGatewayUnavailable(code) {
-    const firstMark = this.#gatewayUnavailableSince === undefined
-    this.#gatewayUnavailableSince = Date.now()
+    const firstMark = this.#gatewayDownSince === undefined
+    this.#gatewayDownSince ??= Date.now()
     if (firstMark) {
       logDiagnostic(this.#options.logger, "reverse_client.gateway_unavailable", {
         code,
         generation: this.#generation,
         ...this.#routeIdentityFields()
       })
+      this.#emitStateChange()
     }
+    this.#scheduleGatewayProbe()
   }
 
   /**
-   * Whether the hub last reported this member's route class as having no selectable
-   * gateway. The report is a property of the live connection, not a short-lived cache:
-   * it arrives once per connection at selection time and only changes when the hub says
-   * otherwise or an open succeeds. Expiring it on a timer made the transport set pick a
-   * route class the hub had already declared unserviceable, and the reset that followed
-   * failed the caller's operation — a build lost its first transport open this way.
-   * The transport set still falls back to marked members when every member is marked, so
-   * a mark can never wedge the set.
-   * @returns {boolean} True while the hub reports no selectable gateway for this member.
+   * Registers this member's route class as available again, which is the only
+   * transition back into service.
+   * @returns {void}
+   */
+  #markGatewayAvailable() {
+    if (this.#gatewayDownSince === undefined) return
+    const downMs = Date.now() - this.#gatewayDownSince
+    this.#gatewayDownSince = undefined
+    this.#gatewayProbeDelay = this.#options.gatewayProbeDelayMs ?? DEFAULT_GATEWAY_PROBE_DELAY_MS
+    this.#gatewayProbeAttempts = 0
+    if (this.#gatewayProbeTimer !== undefined) clearTimeout(this.#gatewayProbeTimer)
+    this.#gatewayProbeTimer = undefined
+    logDiagnostic(this.#options.logger, "reverse_client.gateway_available", {
+      downMs,
+      generation: this.#generation,
+      ...this.#routeIdentityFields()
+    })
+    this.#emitStateChange()
+  }
+
+  /**
+   * Keeps reaching the hub while this member is down, so a returning gateway is
+   * detected and taken into service again without operator action. Only a fresh
+   * hub selection can report a route class available, so the probe re-selects.
+   * @returns {void}
+   */
+  #scheduleGatewayProbe() {
+    if (this.#closing || this.#fatal || this.#gatewayDownSince === undefined || this.#gatewayProbeTimer !== undefined) return
+    const delay = this.#gatewayProbeDelay
+    this.#gatewayProbeDelay = Math.min(this.#gatewayProbeDelay * 2, this.#options.maxGatewayProbeDelayMs ?? DEFAULT_MAX_GATEWAY_PROBE_DELAY_MS)
+    this.#gatewayProbeTimer = setTimeout(() => {
+      this.#gatewayProbeTimer = undefined
+      this.#probeGateway()
+    }, delay)
+    if (!this.#referenced) this.#gatewayProbeTimer.unref?.()
+  }
+
+  /** Re-runs hub selection for a down route class and re-reads its availability. @returns {void} */
+  #probeGateway() {
+    if (this.#closing || this.#fatal || this.#gatewayDownSince === undefined) return
+    this.#gatewayProbeAttempts += 1
+    logDiagnostic(this.#options.logger, "reverse_client.gateway_probe", {
+      attempt: this.#gatewayProbeAttempts,
+      generation: this.#generation,
+      physicalState: this.#state,
+      ...this.#routeIdentityFields()
+    })
+    const physical = this.#physical
+    if (physical !== undefined && !physical.destroyed) {
+      physical.destroy()
+      return
+    }
+    if (this.#connectPromise === undefined) {
+      this.#connectPromise = new Promise((resolve, reject) => { this.#resolveConnect = resolve; this.#rejectConnect = reject })
+      this.#connectPromise.catch(() => {})
+    }
+    this.#connectPhysical()
+  }
+
+  /** Notifies owners that availability changed without changing the physical state. @returns {void} */
+  #emitStateChange() {
+    this.emit("stateChange", this.#state, undefined)
+  }
+
+  /**
+   * Whether the hub currently has this member's route class registered down. The
+   * state never expires on its own: a down member stays out of service until the
+   * hub reports a selectable gateway again.
+   * @returns {boolean} True while the route class is registered down.
    */
   #gatewayUnavailable() {
-    return this.#gatewayUnavailableSince !== undefined
+    return this.#gatewayDownSince !== undefined
   }
 
   status() {
@@ -393,6 +463,7 @@ export class ReverseGatewayConnection extends EventEmitter {
     return {
       physicalState: this.#state,
       gatewayUnavailable: this.#gatewayUnavailable(),
+      gatewayProbeAttempts: this.#gatewayProbeAttempts,
       connectionGeneration: this.#generation,
       virtualStreams: this.#streams.size,
       streamStates,
@@ -421,6 +492,8 @@ export class ReverseGatewayConnection extends EventEmitter {
     this.#closing = true
     if (this.#reconnectTimer !== undefined) clearTimeout(this.#reconnectTimer)
     this.#reconnectTimer = undefined
+    if (this.#gatewayProbeTimer !== undefined) clearTimeout(this.#gatewayProbeTimer)
+    this.#gatewayProbeTimer = undefined
     this.#setState("closed")
     for (const stream of [...this.#streams.values()]) stream.socket.destroy()
     const physical = this.#physical
@@ -446,6 +519,8 @@ export class ReverseGatewayConnection extends EventEmitter {
     this.#closing = true
     if (this.#reconnectTimer !== undefined) clearTimeout(this.#reconnectTimer)
     this.#reconnectTimer = undefined
+    if (this.#gatewayProbeTimer !== undefined) clearTimeout(this.#gatewayProbeTimer)
+    this.#gatewayProbeTimer = undefined
     this.#setState("closed")
     const physical = this.#physical
     physical?.pause()
@@ -658,7 +733,7 @@ export class ReverseGatewayConnection extends EventEmitter {
             const selection = this.#routeIdentityFields()
             if (detail.generation !== generation || ("gatewayId" in selection ? detail.gatewayId !== selection.gatewayId || Object.hasOwn(detail, "routeClass") : detail.routeClass !== selection.routeClass || Object.hasOwn(detail, "gatewayId"))) throw fatalError("GATEWAY_SELECTION_FAILED")
             if (typeof detail.resumed !== "boolean") throw fatalError("GATEWAY_SELECTION_FAILED")
-            if (detail.selectable === true) this.#gatewayUnavailableSince = undefined
+            if (detail.selectable === true) this.#markGatewayAvailable()
             else if (detail.selectable === false) this.#markGatewayUnavailable("GATEWAY_SELECT_UNAVAILABLE")
             selected = true
             if (detail.resumed) {
@@ -785,13 +860,7 @@ export class ReverseGatewayConnection extends EventEmitter {
       stream.sendWindow = Math.min(stream.sendWindow, detail.initialWindow)
       stream.state = stream.localEnded ? "half-closed" : "open"
       stream.opened = true
-      if (this.#gatewayUnavailableSince !== undefined) {
-        this.#gatewayUnavailableSince = undefined
-        logDiagnostic(this.#options.logger, "reverse_client.gateway_available", {
-          generation: this.#generation,
-          ...this.#routeIdentityFields()
-        })
-      }
+      this.#markGatewayAvailable()
       logDiagnostic(this.#options.logger, "virtual_stream.opened", {
         streamId: stream.id,
         generation: this.#generation,
@@ -1336,6 +1405,9 @@ function validateOptions(options) {
   positiveInteger(options.maxConnectionBufferBytes, "maxConnectionBufferBytes", DEFAULT_CONNECTION_BUFFER_BYTES)
   if (options.spoolDirectory !== undefined && (typeof options.spoolDirectory !== "string" || options.spoolDirectory.length === 0)) throw new TypeError("spoolDirectory must be a non-empty string")
   positiveInteger(options.maxAgeMs, "maxAgeMs", 24 * 60 * 60 * 1000)
+  const gatewayProbeDelay = positiveInteger(options.gatewayProbeDelayMs, "gatewayProbeDelayMs", DEFAULT_GATEWAY_PROBE_DELAY_MS)
+  const maxGatewayProbeDelay = positiveInteger(options.maxGatewayProbeDelayMs, "maxGatewayProbeDelayMs", DEFAULT_MAX_GATEWAY_PROBE_DELAY_MS)
+  if (maxGatewayProbeDelay < gatewayProbeDelay) throw new RangeError("maxGatewayProbeDelayMs must not be smaller than gatewayProbeDelayMs")
   if (options.initialGeneration !== undefined && (!Number.isSafeInteger(options.initialGeneration) || options.initialGeneration < 0)) throw new RangeError("initialGeneration must be a non-negative safe integer")
 }
 

package/test/integration/reverse-route-class.test.js

@@ -56,6 +56,15 @@ async function waitUntil(predicate, label) {
   }
 }
 
+/** @param {() => boolean} predicate - Completion predicate. @param {string} label - Failure label. @param {number} timeoutMs - Deadline. */
+async function waitUntilWithin(predicate, label, timeoutMs) {
+  const deadline = Date.now() + timeoutMs
+  while (!predicate()) {
+    if (Date.now() >= deadline) throw new Error(`Timed out: ${label}`)
+    await new Promise((resolve) => setTimeout(resolve, 10))
+  }
+}
+
 /** @param {string} prefix - Echo response prefix. */
 async function createEchoTarget(prefix) {
   const sockets = new Set()
@@ -470,4 +479,74 @@ describe("reverse gateway route-class", () => {
     expect(await exchange(stream, "probe")).toBe("a:probe")
     stream.destroy()
   }))
+
+  it("a route class registered down stays down, keeps being reached, and rejoins automatically", async () => withTestCleanup(async (after) => {
+    const root = await mkdtemp(join(tmpdir(), "socketduct-route-class-down-"))
+    after(() => rm(root, {recursive: true, force: true}))
+    const certificates = generateTestCertificates(root)
+    const targetA = await createEchoTarget("a:")
+    after(() => targetA.close())
+
+    const hub = await startReverseGatewayHub({
+      tokens: new StaticTokenStore([
+        {username: "home-a", token: "gateway-a-token"},
+        {username: "tensorbuzz", token: "client-token"}
+      ]),
+      tls: {certFile: certificates.serverCertFile, keyFile: certificates.serverKeyFile},
+      knownGatewayIds: ["server-a-starlink"],
+      knownRouteClasses: ["starlink"],
+      authorizeGateway: ({principal, gatewayId}) => principal.username === "home-a" && gatewayId === "server-a-starlink",
+      authorizeClientGateway: () => false,
+      authorizeClientRouteClass: ({principal, routeClass}) => principal.username === "tensorbuzz" && routeClass === "starlink",
+      resolveRouteClassGateways: () => ["server-a-starlink"],
+      resolveOpenGateways: ({routeClass, target}) => (routeClass === "starlink" && target.host === targetA.host ? ["server-a-starlink"] : []),
+      authorizeOpen: ({gatewayId, target}) => gatewayId === "server-a-starlink" && target.port === targetA.port
+    })
+    after(() => hub.close())
+    const tls = {caFile: certificates.caCertFile, servername: "localhost"}
+
+    const client = new ReverseGatewayConnection({
+      routeClass: "starlink",
+      connectionId: "route_class_down_client",
+      hub: {host: "127.0.0.1", port: hub.port, token: "client-token"},
+      tls,
+      gatewayProbeDelayMs: 20,
+      maxGatewayProbeDelayMs: 40,
+      reconnectDelayMs: 10,
+      maxReconnectDelayMs: 20
+    })
+    after(() => client.close())
+
+    await client.connect()
+    expect(client.status().gatewayUnavailable).toBe(true)
+
+    // The member keeps reaching the hub while its route class is down.
+    await waitUntilWithin(() => client.status().gatewayProbeAttempts >= 3, "gateway re-reaching", 4_000)
+
+    // A down registration never expires on its own: the previous five-second
+    // mark would have silently returned this member to service with no gateway.
+    await new Promise((resolve) => setTimeout(resolve, 5_100))
+    expect(client.status().gatewayUnavailable).toBe(true)
+    expect(client.status().gatewayProbeAttempts >= 3).toBe(true)
+
+    const gatewayA = new OutboundReverseGateway({
+      gatewayId: "server-a-starlink",
+      connectionId: "down_gateway_a",
+      hub: {host: "127.0.0.1", port: hub.port, token: "gateway-a-token"},
+      tls,
+      authorizeTarget: (target) => target.host === targetA.host && target.port === targetA.port,
+      reconnectDelayMs: 10,
+      maxReconnectDelayMs: 20
+    })
+    after(() => gatewayA.close())
+    await gatewayA.start()
+    await waitUntil(() => hub.status().healthyGateways === 1, "gateway A registration")
+
+    // The same member registers the class up again and is taken back into use.
+    await waitUntilWithin(() => client.status().gatewayUnavailable === false, "automatic re-registration", 4_000)
+    const stream = client.createConnection({host: targetA.host, port: targetA.port})
+    await onceEvent(stream, "connect")
+    expect(await exchange(stream, "probe")).toBe("a:probe")
+    stream.destroy()
+  }))
 })

package/test/unit/client/multiplexed-transport-set.test.js

@@ -552,16 +552,30 @@ describe("multiplexed transport set", () => {
     await set.close()
   })
 
-  it("a stale gateway-unavailable mark cannot wedge the set when every ready member is marked", async () => {
+  it("a member the hub registered down is never selected, even when every member is down", async () => {
     const {set, members} = fixture()
     const started = set.start()
     members.forEach((member) => member.ready())
     await started
     members.forEach((member) => { member.gatewayUnavailable = true })
+    let callbacks = 0
     const socket = set.createConnection({host: "target.internal", port: 443}, (error) => {
       expect(error).toBe(null)
+      callbacks += 1
     })
+    await new Promise((resolve) => setImmediate(resolve))
+    expect(callbacks).toBe(0)
+    expect(socket.gatewayId).toBe(undefined)
+    expect(set.status().pendingOpens).toBe(1)
+    expect(members.map((member) => member.opens)).toEqual([0, 0, 0])
+    expect(set.status().members.map((member) => member.gatewayUnavailable)).toEqual([true, true, true])
+
+    at(members, 2).gatewayUnavailable = false
+    at(members, 2).setState("ready")
+    await new Promise((resolve) => setImmediate(resolve))
+    expect(callbacks).toBe(1)
     expect(socket.gatewayId !== undefined).toBe(true)
+    expect(members.map((member) => member.opens)).toEqual([0, 0, 1])
     expect(set.status().pendingOpens).toBe(0)
     await set.close()
   })