socketduct 0.1.24 → 0.1.25

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "socketduct",
-  "version": "0.1.24",
+  "version": "0.1.25",
   "description": "Resilient authenticated tunnel/proxy for keeping socket streams alive across flaky links.",
   "type": "module",
   "main": "./src/index.js",

package/src/client/multiplexed-transport-set.js

@@ -44,6 +44,7 @@ const GATEWAY_ID = /^[A-Za-z0-9][A-Za-z0-9_-]{1,63}$/
  * @property {import('../logging/structured-logger.js').DiagnosticLogger} [logger] - Optional privacy-safe structured logger shared by the set and its members.
  * @typedef {object} MemberConnectionStatus
  * @property {string} physicalState - Member lifecycle.
+ * @property {boolean} [gatewayUnavailable] - Hub recently reported no selectable gateway for this member's route class.
  * @property {number} connectionGeneration - Current authenticated generation.
  * @property {number} physicalConnects - Authenticated physical generations.
  * @property {number} reconnectAttempts - Physical reconnect attempts.
@@ -569,17 +570,34 @@ export class MultiplexedSocketductTransportSet extends EventEmitter {
     return this.#closePromise
   }
 
-  /** @returns {{member: TransportMember, previousCursor: number} | undefined} */
+  /**
+   * Selects the next ready member, preferring members whose gateway the hub has
+   * not recently reported unavailable. When every ready member carries a
+   * gateway-unavailable mark, any ready member may still be selected so a stale
+   * mark can never wedge the set.
+   * @returns {{member: TransportMember, previousCursor: number} | undefined}
+   */
   #selectReadyMember() {
     if (this.#closed) throw closedError()
     const previousCursor = this.#cursor
+    /** @type {{member: TransportMember, index: number} | undefined} */
+    let fallback
     for (let offset = 0; offset < this.#members.length; offset += 1) {
       const index = (this.#cursor + offset) % this.#members.length
       const member = /** @type {TransportMember} */ (this.#members[index])
-      if (member.connection.status().physicalState !== "ready") continue
+      const status = member.connection.status()
+      if (status.physicalState !== "ready") continue
+      if (status.gatewayUnavailable === true) {
+        fallback ??= {member, index}
+        continue
+      }
       this.#cursor = (index + 1) % this.#members.length
       return {member, previousCursor}
     }
+    if (fallback !== undefined) {
+      this.#cursor = (fallback.index + 1) % this.#members.length
+      return {member: fallback.member, previousCursor}
+    }
     return undefined
   }
 

package/src/reverse/client.js

@@ -32,6 +32,7 @@ import {attachDisconnectAttribution, createDisconnectAttribution, parseDisconnec
 const DEFAULT_STREAM_BUFFER_BYTES = 4 * 1024 * 1024
 const DEFAULT_CONNECTION_BUFFER_BYTES = 32 * 1024 * 1024
 const MANIFEST_PAGE_STREAMS = 64
+const GATEWAY_UNAVAILABLE_TTL_MS = 5_000
 
 /**
  * @typedef {object} ReverseGatewayConnectionOptions
@@ -168,6 +169,7 @@ export class ReverseGatewayConnection extends EventEmitter {
   /** @type {number} */ #spoolBytes = 0
   /** @type {number} */ #spoolCleanupFailures = 0
   /** @type {Set<Promise<void>>} */ #spoolCleanupTasks = new Set()
+  /** @type {number | undefined} - Wall-clock time of the last hub-reported gateway unavailability. */ #gatewayUnavailableSince
   /** @type {import('./disconnect-attribution.js').DisconnectAttribution | undefined} */ #disconnectAttribution
 
   /** @param {ReverseGatewayConnectionOptions} options - Hub route and safety policy. */
@@ -352,12 +354,41 @@ export class ReverseGatewayConnection extends EventEmitter {
     return socket
   }
 
+  /**
+   * Records that the hub just reported no selectable gateway for this member's
+   * route class, so the transport set can prefer members that can actually open.
+   * @param {string} code - Hub reset code.
+   * @returns {void}
+   */
+  #markGatewayUnavailable(code) {
+    const firstMark = this.#gatewayUnavailableSince === undefined
+    this.#gatewayUnavailableSince = Date.now()
+    if (firstMark) {
+      logDiagnostic(this.#options.logger, "reverse_client.gateway_unavailable", {
+        code,
+        generation: this.#generation,
+        ...this.#routeIdentityFields()
+      })
+    }
+  }
+
+  /**
+   * Whether the last hub-reported gateway state for this member was unavailable
+   * and the report is still fresh. A stale report never pins the member out.
+   * @returns {boolean} True while a fresh hub unavailability report is active.
+   */
+  #gatewayUnavailable() {
+    return this.#gatewayUnavailableSince !== undefined
+      && Date.now() - this.#gatewayUnavailableSince < GATEWAY_UNAVAILABLE_TTL_MS
+  }
+
   status() {
     /** @type {Record<string, number>} */
     const streamStates = {}
     for (const stream of this.#streams.values()) streamStates[stream.state] = (streamStates[stream.state] ?? 0) + 1
     return {
       physicalState: this.#state,
+      gatewayUnavailable: this.#gatewayUnavailable(),
       connectionGeneration: this.#generation,
       virtualStreams: this.#streams.size,
       streamStates,
@@ -623,6 +654,8 @@ export class ReverseGatewayConnection extends EventEmitter {
             const selection = this.#routeIdentityFields()
             if (detail.generation !== generation || ("gatewayId" in selection ? detail.gatewayId !== selection.gatewayId || Object.hasOwn(detail, "routeClass") : detail.routeClass !== selection.routeClass || Object.hasOwn(detail, "gatewayId"))) throw fatalError("GATEWAY_SELECTION_FAILED")
             if (typeof detail.resumed !== "boolean") throw fatalError("GATEWAY_SELECTION_FAILED")
+            if (detail.selectable === true) this.#gatewayUnavailableSince = undefined
+            else if (detail.selectable === false) this.#markGatewayUnavailable("GATEWAY_SELECT_UNAVAILABLE")
             selected = true
             if (detail.resumed) {
               clearTimeout(timer)
@@ -745,9 +778,16 @@ export class ReverseGatewayConnection extends EventEmitter {
       const detail = decodeV3Control(frame.payload)
       if (stream.state !== "opening" || frame.sequence !== 0 || detail.identifier !== stream.identifier || !validPeerWindow(detail.initialWindow)) throw new Error("Invalid reverse OPENED response")
       stream.sendWindowLimit = Math.min(stream.sendWindowLimit, detail.initialWindow)
-      stream.sendWindow = Math.min(stream.sendWindow, stream.sendWindowLimit)
+      stream.sendWindow = Math.min(stream.sendWindow, detail.initialWindow)
       stream.state = stream.localEnded ? "half-closed" : "open"
       stream.opened = true
+      if (this.#gatewayUnavailableSince !== undefined) {
+        this.#gatewayUnavailableSince = undefined
+        logDiagnostic(this.#options.logger, "reverse_client.gateway_available", {
+          generation: this.#generation,
+          ...this.#routeIdentityFields()
+        })
+      }
       logDiagnostic(this.#options.logger, "virtual_stream.opened", {
         streamId: stream.id,
         generation: this.#generation,
@@ -801,6 +841,9 @@ export class ReverseGatewayConnection extends EventEmitter {
       const detail = decodeV3Control(frame.payload)
       const code = typeof detail.code === "string" ? detail.code : "STREAM_RESET"
       stream.state = "reset"
+      if (code === "GATEWAY_UNAVAILABLE" || code === "GATEWAY_DISCONNECTED" || code === "GATEWAY_REPLACED") {
+        this.#markGatewayUnavailable(code)
+      }
       const error = attachDisconnectAttribution(new SocketductStreamError("Reverse gateway stream was reset", code), validHubAttribution(detail) ?? (code === "GATEWAY_DISCONNECTED" ? this.#disconnectAttribution : undefined))
       stream.socket._receiveError(error)
       return

package/src/reverse/hub.js

@@ -49,6 +49,7 @@ const MANIFEST_PAGE_STREAMS = 64
  * @property {(input: {principal: Principal, gatewayId: string}) => boolean} authorizeGateway - Stable gateway identity binding.
  * @property {(input: {principal: Principal, gatewayId: string}) => boolean} authorizeClientGateway - Client-to-gateway selection policy.
  * @property {(input: {principal: Principal, routeClass: string}) => boolean} [authorizeClientRouteClass] - Client-to-route-class selection policy.
+ * @property {(input: {principal: Principal, routeClass: string}) => string[]} [resolveRouteClassGateways] - All configured gateways for a client route class, used to announce route-class availability at selection time.
  * @property {(input: {principal: Principal, gatewayId: string, target: Target}) => boolean} authorizeOpen - Hub-side route policy.
  * @property {(input: {principal: Principal, routeClass: string, target: Target}) => string[]} [resolveOpenGateways] - Eligible exact gateways for a route-class OPEN.
  * @property {string[]} [knownGatewayIds] - Configured IDs that may be temporarily unavailable.
@@ -480,7 +481,15 @@ export async function startReverseGatewayHub(/** @type {ReverseGatewayHubOptions
     if (clientOwner.gatewayId !== undefined || (clientOwner.routeClass !== undefined && clientOwner.routeClass !== request.routeClass)) throw protocolError("GATEWAY_SELECTION_MISMATCH")
     physical.routeClass = request.routeClass
     clientOwner.routeClass = request.routeClass
-    await sendControl(physical, V3FrameType.GATEWAY_SELECTED, 0, physical.generation, {routeClass: request.routeClass, generation: physical.generation, resumed: physical.resumeRequired})
+    const configuredGatewayIds = options.resolveRouteClassGateways?.({principal: physical.principal, routeClass: request.routeClass})
+    let selectable
+    if (Array.isArray(configuredGatewayIds) && configuredGatewayIds.length > 0) {
+      selectable = configuredGatewayIds.some((gatewayId) => {
+        const owner = gateways.get(gatewayId)
+        return owner !== undefined && owner.physical !== undefined && !owner.physical.socket.destroyed && !owner.physical.closing && !owner.physical.resumeRequired
+      })
+    }
+    await sendControl(physical, V3FrameType.GATEWAY_SELECTED, 0, physical.generation, {routeClass: request.routeClass, generation: physical.generation, resumed: physical.resumeRequired, ...(selectable === undefined ? {} : {selectable})})
     armResumeTimeout(physical)
   }
 

package/src/reverse/runtime.js

@@ -27,7 +27,15 @@ export async function startReverseHubRuntime(config, logger) {
     gatewayIds.push(route.gatewayId)
     routeClassGateways.set(key, gatewayIds)
   }
-  const routeClassRows = new Set(routeClassRoutes.map((route) => `${formatRouteClassRoute(route.username, route.routeClass, {host: route.host, port: route.port})}\0${route.gatewayId}`))
+  const routeClassRows = new Set(routeClassRoutes.map((route) => `${formatRouteClassRoute(route.username, route.routeClass, {host: route.host, port: route.port})}\\0${route.gatewayId}`))
+  /** @type {Map<string, string[]>} - Every configured gateway for a (username, routeClass) pair, independent of target. */
+  const routeClassGatewayUnions = new Map()
+  for (const route of routeClassRoutes) {
+    const key = `${route.username}\\0${route.routeClass}`
+    const gatewayIds = routeClassGatewayUnions.get(key) ?? []
+    if (!gatewayIds.includes(route.gatewayId)) gatewayIds.push(route.gatewayId)
+    routeClassGatewayUnions.set(key, gatewayIds)
+  }
   if (bindings.size !== config.reverseGatewayBindings.length) throw new Error("SOCKETDUCT_REVERSE_GATEWAY_BINDINGS contains a duplicate gateway ID")
   if (exactRouteKeys.size !== exactRoutes.length) throw new Error("SOCKETDUCT_REVERSE_ROUTES contains a duplicate route")
   if (routeClassRows.size !== routeClassRoutes.length) throw new Error("SOCKETDUCT_REVERSE_ROUTE_CLASS_ROUTES contains a duplicate route")
@@ -47,7 +55,8 @@ export async function startReverseHubRuntime(config, logger) {
     authorizeGateway: ({principal, gatewayId}) => bindings.get(gatewayId) === principal.username,
     authorizeClientGateway: ({principal, gatewayId}) => exactSelections.has(`${principal.username}\0${gatewayId}`),
     ...(knownRouteClasses.length === 0 ? {} : {
-      authorizeClientRouteClass: ({principal, routeClass}) => routeClassSelections.has(`${principal.username}\0${routeClass}`),
+      authorizeClientRouteClass: ({principal, routeClass}) => routeClassSelections.has(`${principal.username}\u0000${routeClass}`),
+      resolveRouteClassGateways: ({principal, routeClass}) => routeClassGatewayUnions.get(`${principal.username}\u0000${routeClass}`) ?? [],
       resolveOpenGateways: ({principal, routeClass, target}) => [...(routeClassGateways.get(formatRouteClassRoute(principal.username, routeClass, target)) ?? [])]
     }),
     authorizeOpen: ({principal, gatewayId, target}) => authorizedRoutes.has(formatRoute(principal.username, gatewayId, target)),

package/test/integration/reverse-route-class.test.js

@@ -380,3 +380,71 @@ test("a backpressured gateway does not block DATA or a new OPEN for another rout
   streamA.destroy()
   streamB.destroy()
 })
+
+test("hub announces route-class availability at selection and the client exposes gateway unavailability", async (t) => {
+  const root = await mkdtemp(join(tmpdir(), "socketduct-route-class-selectable-"))
+  t.after(() => rm(root, {recursive: true, force: true}))
+  const certificates = generateTestCertificates(root)
+  const targetA = await createEchoTarget("a:")
+  t.after(() => targetA.close())
+
+  const hub = await startReverseGatewayHub({
+    tokens: new StaticTokenStore([
+      {username: "home-a", token: "gateway-a-token"},
+      {username: "tensorbuzz", token: "client-token"}
+    ]),
+    tls: {certFile: certificates.serverCertFile, keyFile: certificates.serverKeyFile},
+    knownGatewayIds: ["server-a-starlink"],
+    knownRouteClasses: ["starlink"],
+    authorizeGateway: ({principal, gatewayId}) => principal.username === "home-a" && gatewayId === "server-a-starlink",
+    authorizeClientGateway: () => false,
+    authorizeClientRouteClass: ({principal, routeClass}) => principal.username === "tensorbuzz" && routeClass === "starlink",
+    resolveRouteClassGateways: () => ["server-a-starlink"],
+    resolveOpenGateways: ({routeClass, target}) => (routeClass === "starlink" && target.host === targetA.host ? ["server-a-starlink"] : []),
+    authorizeOpen: ({gatewayId, target}) => gatewayId === "server-a-starlink" && target.port === targetA.port
+  })
+  t.after(() => hub.close())
+  const tls = {caFile: certificates.caCertFile, servername: "localhost"}
+
+  const client = new ReverseGatewayConnection({
+    routeClass: "starlink",
+    connectionId: "route_class_selectable_client",
+    hub: {host: "127.0.0.1", port: hub.port, token: "client-token"},
+    tls
+  })
+  t.after(() => client.close())
+
+  // No gateway registered for the route class yet: the hub must announce selectable=false
+  // and the client must surface the member as gateway-unavailable before any open is attempted.
+  await client.connect()
+  assert.equal(client.status().physicalState, "ready")
+  assert.equal(client.status().gatewayUnavailable, true)
+
+  // Once the gateway registers, a replacement selection announces selectable=true and clears the mark.
+  const gatewayA = new OutboundReverseGateway({
+    gatewayId: "server-a-starlink",
+    connectionId: "selectable_gateway_a",
+    hub: {host: "127.0.0.1", port: hub.port, token: "gateway-a-token"},
+    tls,
+    authorizeTarget: (target) => target.host === targetA.host && target.port === targetA.port,
+    reconnectDelayMs: 10,
+    maxReconnectDelayMs: 20
+  })
+  t.after(() => gatewayA.close())
+  await gatewayA.start()
+  await waitUntil(() => hub.status().healthyGateways === 1, "gateway A registration")
+
+  const replacement = new ReverseGatewayConnection({
+    routeClass: "starlink",
+    connectionId: "selectable_client_replacement",
+    hub: {host: "127.0.0.1", port: hub.port, token: "client-token"},
+    tls
+  })
+  t.after(() => replacement.close())
+  await replacement.connect()
+  assert.equal(replacement.status().gatewayUnavailable, false)
+  const stream = replacement.createConnection({host: targetA.host, port: targetA.port})
+  await onceEvent(stream, "connect")
+  assert.equal(await exchange(stream, "probe"), "a:probe")
+  stream.destroy()
+})

package/test/unit/client/multiplexed-transport-set.test.js

@@ -71,6 +71,7 @@ class FakeMember extends EventEmitter {
     this.state = "configured"
     this.generation = 0
     this.physicalConnects = 0
+    this.gatewayUnavailable = false
     /** @type {Promise<void>} */
     this.connectPromise = new Promise((resolve, reject) => {
       this.resolveConnect = resolve
@@ -135,6 +136,7 @@ class FakeMember extends EventEmitter {
   status() {
     return {
       physicalState: this.state,
+      gatewayUnavailable: this.gatewayUnavailable,
       connectionGeneration: this.generation,
       physicalConnects: this.physicalConnects,
       reconnectAttempts: this.state === "reconnecting" ? 1 : 0,
@@ -524,6 +526,35 @@ test("25 and 100 continuously healthy opens have exact deterministic distributio
   }
 })
 
+test("opens skip a ready member whose gateway the hub reported unavailable", async () => {
+  const {set, members} = fixture()
+  const started = set.start()
+  members.forEach((member) => member.ready())
+  await started
+  at(members, 0).gatewayUnavailable = true
+  for (let offset = 0; offset < 12; offset += 1) {
+    const socket = set.createConnection({host: "target.internal", port: 443})
+    assert.equal(gatewayOf(socket) === "starlink", false)
+  }
+  assert.deepEqual(set.status().members.map((member) => member.assignments), [0, 6, 6])
+  at(members, 0).gatewayUnavailable = false
+  const recovered = set.createConnection({host: "target.internal", port: 443})
+  assert.equal(gatewayOf(recovered), "starlink")
+  await set.close()
+})
+
+test("a stale gateway-unavailable mark cannot wedge the set when every ready member is marked", async () => {
+  const {set, members} = fixture()
+  const started = set.start()
+  members.forEach((member) => member.ready())
+  await started
+  members.forEach((member) => { member.gatewayUnavailable = true })
+  const socket = set.createConnection({host: "target.internal", port: 443}, (error) => assert.equal(error, null))
+  assert.equal(socket.gatewayId !== undefined, true)
+  assert.equal(set.status().pendingOpens, 0)
+  await set.close()
+})
+
 test("all-members reconnecting queues FIFO opens until one ready member recovers", async () => {
   const {set, members} = fixture()
   const started = set.start()