socketduct 0.1.28 → 0.1.30

package/changelog.d/20261025123000-route-class-down-registration.md

@@ -0,0 +1 @@
+A route class the hub reports down is never selected while it stays down: the member keeps re-reaching the hub on a bounded backoff and returns to service automatically once a gateway registers again, and `status()` now exposes the down state and the probe attempts.

package/changelog.d/20261025193000-drain-transport-without-resetting-streams.md

@@ -0,0 +1 @@
+Multiplexed connections gain `drain()`, which refuses new virtual streams, waits for the open ones to finish, and only then releases the physical transport, so a shutdown no longer resets work that is still in flight; `close()` keeps its immediate-termination behaviour and now logs how many live streams it terminated.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "socketduct",
-  "version": "0.1.28",
+  "version": "0.1.30",
   "description": "Resilient authenticated tunnel/proxy for keeping socket streams alive across flaky links.",
   "type": "module",
   "main": "./src/index.js",

package/src/client/multiplexed-connection.js

@@ -138,6 +138,11 @@ export class MultiplexedSocketductConnection extends EventEmitter {
 
   /** @type {boolean} */
   #closing = false
+  #draining = false
+  /** @type {Promise<void> | undefined} */
+  #closePromise = undefined
+  /** @type {Array<() => void>} */
+  #drainWaiters = []
 
   /** @type {boolean} */
   #warm = false
@@ -247,7 +252,7 @@ export class MultiplexedSocketductConnection extends EventEmitter {
    * @returns {Promise<void>} Resolves when the member becomes ready.
    */
   connect() {
-    if (this.#closing) return Promise.reject(Object.assign(new Error("Multiplexed Socketduct connection is closed"), {code: "SOCKETDUCT_CONNECTION_CLOSED"}))
+    if (this.#closing || this.#draining) return Promise.reject(Object.assign(new Error("Multiplexed Socketduct connection is closed"), {code: "SOCKETDUCT_CONNECTION_CLOSED"}))
     if (this.#warmPromise !== undefined) return this.#warmPromise
     this.#warm = true
     this.#warmPromise = Promise.all([...this.#streams.values()].map((stream) => stream.spoolReady)).then(() => this.#ensureConnected())
@@ -297,7 +302,7 @@ export class MultiplexedSocketductConnection extends EventEmitter {
    * @returns {VirtualSocket} Socket-shaped Duplex.
    */
   #createConnection(options, callback, connectOnReady) {
-    if (this.#closing) throw new Error("Multiplexed Socketduct connection is closed")
+    if (this.#closing || this.#draining) throw new Error("Multiplexed Socketduct connection is closed")
     if (options.manualReceiveAck !== undefined) throw new TypeError("manualReceiveAck is supported only by reverse gateway connections")
     if (this.#streams.size >= (this.#options.maxStreams ?? DEFAULT_MAX_STREAMS)) throw new Error("Maximum virtual stream count exceeded")
     const target = normalizeTarget(options)
@@ -446,10 +451,52 @@ export class MultiplexedSocketductConnection extends EventEmitter {
     return this
   }
 
-  /** Gracefully closes all virtual streams and the shared transport. @returns {Promise<void>} */
+  /**
+   * Terminates every virtual stream and releases the shared transport. Streams
+   * still in flight are reset at the relay by design, so the number of them is
+   * logged (`multiplexed_connection.shutdown_with_live_streams`) instead of
+   * disappearing. Use `drain()` when in-flight work must survive.
+   * @returns {Promise<void>}
+   */
   async close() {
+    if (this.#closePromise !== undefined) return this.#closePromise
+    this.#closePromise = this.#shutdownTransport()
+
+    return this.#closePromise
+  }
+
+  /**
+   * Graceful counterpart of `close()`: refuses new virtual streams, lets the
+   * open ones finish, and only then releases the shared transport. Use this
+   * wherever a transport must not reset work that is still in flight.
+   * @returns {Promise<void>}
+   */
+  async drain() {
+    if (this.#closePromise !== undefined) return this.#closePromise
+    this.#draining = true
+    this.#closePromise = (async () => {
+      await this.#waitForStreamsToDrain()
+      await this.#shutdownTransport()
+    })()
+
+    return this.#closePromise
+  }
+
+  /** @returns {Promise<void>} */
+  #waitForStreamsToDrain() {
+    if (this.#streams.size === 0) return Promise.resolve()
+
+    return new Promise((resolve) => { this.#drainWaiters.push(resolve) })
+  }
+
+  /** @returns {Promise<void>} */
+  async #shutdownTransport() {
     if (this.#closing) return
     this.#closing = true
+
+    if (this.#streams.size > 0) {
+      logDiagnostic(this.#options.logger, "multiplexed_connection.shutdown_with_live_streams", {streamsKilled: this.#streams.size})
+    }
     this.#setPhysicalState("closed")
     if (this.#reconnectTimer !== undefined) clearTimeout(this.#reconnectTimer)
     this.#reconnectTimer = undefined
@@ -618,6 +665,8 @@ export class MultiplexedSocketductConnection extends EventEmitter {
       this.#trackSpoolCleanup(stream)
     }
     if (this.#streams.size === 0 && !this.#authenticated && this.#physical !== undefined) this.#physical.destroy()
+
+    if (this.#streams.size === 0) for (const resolve of this.#drainWaiters.splice(0)) resolve()
   }
 
   /** @param {number} streamId - Stream. @returns {void} */

package/src/client/multiplexed-transport-set.js

@@ -61,6 +61,7 @@ const GATEWAY_ID = /^[A-Za-z0-9][A-Za-z0-9_-]{1,63}$/
  * @property {() => MemberConnectionStatus} status - Privacy-safe status.
  * @property {() => Promise<void>} close - Close the member.
  * @property {() => Promise<void>} detach - Preserve durable streams for process-replacement recovery.
+ * @property {() => Promise<void>} drain - Let live streams finish before releasing the member.
  * @property {(eventName: string, listener: (...args: unknown[]) => void) => unknown} on - Subscribe to lifecycle changes.
  * @property {() => unknown} [ref] - Reference resources.
  * @property {() => unknown} [unref] - Unreference resources.
@@ -493,13 +494,14 @@ export class MultiplexedSocketductTransportSet extends EventEmitter {
     return this
   }
 
-  /** @returns {{state: string, configuredMembers: number, connectingMembers: number, readyMembers: number, degradedMembers: number, closedMembers: number, minimumReadyMembers: number, pendingOpens: number, maxPendingOpens: number, virtualOpens: number, physicalConnects: number, reconnectAttempts: number, budgets: {dataBytes: number, spoolBytes: number, queuedBytes: number, maxDataBytes: number, maxSpoolBytes: number, maxQueuedBytes: number}, members: Array<{gatewayId?: string, routeClass?: string, state: string, assignments: number, physicalConnects: number, reconnectAttempts: number, reconnectSuccesses: number, virtualStreams: number, queuedBytes: number, spoolBytes: number, spoolCleanupFailures: number, terminalErrorCode: string | null}>}} Privacy-safe aggregate and per-member status. */
+  /** @returns {{state: string, configuredMembers: number, connectingMembers: number, readyMembers: number, degradedMembers: number, closedMembers: number, minimumReadyMembers: number, pendingOpens: number, maxPendingOpens: number, virtualOpens: number, physicalConnects: number, reconnectAttempts: number, budgets: {dataBytes: number, spoolBytes: number, queuedBytes: number, maxDataBytes: number, maxSpoolBytes: number, maxQueuedBytes: number}, members: Array<{gatewayId?: string, routeClass?: string, state: string, gatewayUnavailable: boolean, assignments: number, physicalConnects: number, reconnectAttempts: number, reconnectSuccesses: number, virtualStreams: number, queuedBytes: number, spoolBytes: number, spoolCleanupFailures: number, terminalErrorCode: string | null}>}} Privacy-safe aggregate and per-member status. */
   status() {
     const members = this.#members.map((member) => {
       const status = member.connection.status()
       return {
         ...memberAllocation(member),
         state: status.physicalState,
+        gatewayUnavailable: status.gatewayUnavailable === true,
         assignments: member.assignments,
         physicalConnects: status.physicalConnects,
         reconnectAttempts: status.reconnectAttempts,
@@ -549,7 +551,12 @@ export class MultiplexedSocketductTransportSet extends EventEmitter {
     return this.#shutdown("detach")
   }
 
-  /** @param {"close" | "detach"} method - Member shutdown operation. @returns {Promise<void>} */
+  /** Drains live virtual streams on every member before releasing them once. @returns {Promise<void>} */
+  drain() {
+    return this.#shutdown("drain")
+  }
+
+  /** @param {"close" | "detach" | "drain"} method - Member shutdown operation. @returns {Promise<void>} */
   #shutdown(method) {
     if (this.#closePromise !== undefined) return this.#closePromise
     this.#closed = true
@@ -571,33 +578,25 @@ export class MultiplexedSocketductTransportSet extends EventEmitter {
   }
 
   /**
-   * Selects the next ready member, preferring members whose gateway the hub has
-   * not recently reported unavailable. When every ready member carries a
-   * gateway-unavailable mark, any ready member may still be selected so a stale
-   * mark can never wedge the set.
+   * Selects the next ready member whose route class the hub has registered up.
+   * A member the hub registered down is never selected: nothing is sent to a
+   * route class without a reachable gateway. When every member is registered
+   * down the open stays pending here until a member reports availability again
+   * (the member keeps probing the hub, and its state change re-drains this set).
    * @returns {{member: TransportMember, previousCursor: number} | undefined}
    */
   #selectReadyMember() {
     if (this.#closed) throw closedError()
     const previousCursor = this.#cursor
-    /** @type {{member: TransportMember, index: number} | undefined} */
-    let fallback
     for (let offset = 0; offset < this.#members.length; offset += 1) {
       const index = (this.#cursor + offset) % this.#members.length
       const member = /** @type {TransportMember} */ (this.#members[index])
       const status = member.connection.status()
       if (status.physicalState !== "ready") continue
-      if (status.gatewayUnavailable === true) {
-        fallback ??= {member, index}
-        continue
-      }
+      if (status.gatewayUnavailable === true) continue
       this.#cursor = (index + 1) % this.#members.length
       return {member, previousCursor}
     }
-    if (fallback !== undefined) {
-      this.#cursor = (fallback.index + 1) % this.#members.length
-      return {member: fallback.member, previousCursor}
-    }
     return undefined
   }
 

package/src/reverse/client.js

@@ -32,6 +32,8 @@ import {attachDisconnectAttribution, createDisconnectAttribution, parseDisconnec
 const DEFAULT_STREAM_BUFFER_BYTES = 4 * 1024 * 1024
 const DEFAULT_CONNECTION_BUFFER_BYTES = 32 * 1024 * 1024
 const MANIFEST_PAGE_STREAMS = 64
+const DEFAULT_GATEWAY_PROBE_DELAY_MS = 1_000
+const DEFAULT_MAX_GATEWAY_PROBE_DELAY_MS = 30_000
 
 /**
  * @typedef {object} ReverseGatewayConnectionOptions
@@ -50,6 +52,8 @@ const MANIFEST_PAGE_STREAMS = 64
  * @property {number} [connectTimeoutMs] - TLS/auth deadline.
  * @property {number} [reconnectDelayMs] - Initial reconnect delay.
  * @property {number} [maxReconnectDelayMs] - Maximum reconnect delay.
+ * @property {number} [gatewayProbeDelayMs] - Initial re-reach delay while the hub reports no selectable gateway.
+ * @property {number} [maxGatewayProbeDelayMs] - Maximum re-reach delay while the hub reports no selectable gateway.
  * @property {number} [initialGeneration] - Monotonic seed for controlled replacement; defaults to wall-clock microseconds.
  * @property {boolean} [continuousReconnect] - Keep the member warm after loss, default true.
  * @property {import('../protocol/compression.js').CompressionLimits} [compressionLimits] - zstd safety bounds.
@@ -168,7 +172,10 @@ export class ReverseGatewayConnection extends EventEmitter {
   /** @type {number} */ #spoolBytes = 0
   /** @type {number} */ #spoolCleanupFailures = 0
   /** @type {Set<Promise<void>>} */ #spoolCleanupTasks = new Set()
-  /** @type {number | undefined} - Wall-clock time of the last hub-reported gateway unavailability. */ #gatewayUnavailableSince
+  /** @type {number | undefined} - Wall-clock time the hub first reported this member's route class unselectable; undefined means registered available. */ #gatewayDownSince
+  /** @type {ReturnType<typeof setTimeout> | undefined} */ #gatewayProbeTimer
+  /** @type {number} */ #gatewayProbeDelay
+  /** @type {number} */ #gatewayProbeAttempts = 0
   /** @type {import('./disconnect-attribution.js').DisconnectAttribution | undefined} */ #disconnectAttribution
 
   /** @param {ReverseGatewayConnectionOptions} options - Hub route and safety policy. */
@@ -184,6 +191,7 @@ export class ReverseGatewayConnection extends EventEmitter {
     })
     this.#generation = options.initialGeneration ?? Date.now() * 1000
     this.#reconnectDelay = options.reconnectDelayMs ?? 250
+    this.#gatewayProbeDelay = options.gatewayProbeDelayMs ?? DEFAULT_GATEWAY_PROBE_DELAY_MS
     this.#sendWindowLimit = this.#initialWindow()
     this.createConnection = this.createConnection.bind(this)
   }
@@ -360,30 +368,92 @@ export class ReverseGatewayConnection extends EventEmitter {
    * @returns {void}
    */
   #markGatewayUnavailable(code) {
-    const firstMark = this.#gatewayUnavailableSince === undefined
-    this.#gatewayUnavailableSince = Date.now()
+    const firstMark = this.#gatewayDownSince === undefined
+    this.#gatewayDownSince ??= Date.now()
     if (firstMark) {
       logDiagnostic(this.#options.logger, "reverse_client.gateway_unavailable", {
         code,
         generation: this.#generation,
         ...this.#routeIdentityFields()
       })
+      this.#emitStateChange()
     }
+    this.#scheduleGatewayProbe()
   }
 
   /**
-   * Whether the hub last reported this member's route class as having no selectable
-   * gateway. The report is a property of the live connection, not a short-lived cache:
-   * it arrives once per connection at selection time and only changes when the hub says
-   * otherwise or an open succeeds. Expiring it on a timer made the transport set pick a
-   * route class the hub had already declared unserviceable, and the reset that followed
-   * failed the caller's operation — a build lost its first transport open this way.
-   * The transport set still falls back to marked members when every member is marked, so
-   * a mark can never wedge the set.
-   * @returns {boolean} True while the hub reports no selectable gateway for this member.
+   * Registers this member's route class as available again, which is the only
+   * transition back into service.
+   * @returns {void}
+   */
+  #markGatewayAvailable() {
+    if (this.#gatewayDownSince === undefined) return
+    const downMs = Date.now() - this.#gatewayDownSince
+    this.#gatewayDownSince = undefined
+    this.#gatewayProbeDelay = this.#options.gatewayProbeDelayMs ?? DEFAULT_GATEWAY_PROBE_DELAY_MS
+    this.#gatewayProbeAttempts = 0
+    if (this.#gatewayProbeTimer !== undefined) clearTimeout(this.#gatewayProbeTimer)
+    this.#gatewayProbeTimer = undefined
+    logDiagnostic(this.#options.logger, "reverse_client.gateway_available", {
+      downMs,
+      generation: this.#generation,
+      ...this.#routeIdentityFields()
+    })
+    this.#emitStateChange()
+  }
+
+  /**
+   * Keeps reaching the hub while this member is down, so a returning gateway is
+   * detected and taken into service again without operator action. Only a fresh
+   * hub selection can report a route class available, so the probe re-selects.
+   * @returns {void}
+   */
+  #scheduleGatewayProbe() {
+    if (this.#closing || this.#fatal || this.#gatewayDownSince === undefined || this.#gatewayProbeTimer !== undefined) return
+    const delay = this.#gatewayProbeDelay
+    this.#gatewayProbeDelay = Math.min(this.#gatewayProbeDelay * 2, this.#options.maxGatewayProbeDelayMs ?? DEFAULT_MAX_GATEWAY_PROBE_DELAY_MS)
+    this.#gatewayProbeTimer = setTimeout(() => {
+      this.#gatewayProbeTimer = undefined
+      this.#probeGateway()
+    }, delay)
+    if (!this.#referenced) this.#gatewayProbeTimer.unref?.()
+  }
+
+  /** Re-runs hub selection for a down route class and re-reads its availability. @returns {void} */
+  #probeGateway() {
+    if (this.#closing || this.#fatal || this.#gatewayDownSince === undefined) return
+    this.#gatewayProbeAttempts += 1
+    logDiagnostic(this.#options.logger, "reverse_client.gateway_probe", {
+      attempt: this.#gatewayProbeAttempts,
+      generation: this.#generation,
+      physicalState: this.#state,
+      ...this.#routeIdentityFields()
+    })
+    const physical = this.#physical
+    if (physical !== undefined && !physical.destroyed) {
+      physical.destroy()
+      return
+    }
+    if (this.#connectPromise === undefined) {
+      this.#connectPromise = new Promise((resolve, reject) => { this.#resolveConnect = resolve; this.#rejectConnect = reject })
+      this.#connectPromise.catch(() => {})
+    }
+    this.#connectPhysical()
+  }
+
+  /** Notifies owners that availability changed without changing the physical state. @returns {void} */
+  #emitStateChange() {
+    this.emit("stateChange", this.#state, undefined)
+  }
+
+  /**
+   * Whether the hub currently has this member's route class registered down. The
+   * state never expires on its own: a down member stays out of service until the
+   * hub reports a selectable gateway again.
+   * @returns {boolean} True while the route class is registered down.
    */
   #gatewayUnavailable() {
-    return this.#gatewayUnavailableSince !== undefined
+    return this.#gatewayDownSince !== undefined
   }
 
   status() {
@@ -393,6 +463,7 @@ export class ReverseGatewayConnection extends EventEmitter {
     return {
       physicalState: this.#state,
       gatewayUnavailable: this.#gatewayUnavailable(),
+      gatewayProbeAttempts: this.#gatewayProbeAttempts,
       connectionGeneration: this.#generation,
       virtualStreams: this.#streams.size,
       streamStates,
@@ -421,6 +492,8 @@ export class ReverseGatewayConnection extends EventEmitter {
     this.#closing = true
     if (this.#reconnectTimer !== undefined) clearTimeout(this.#reconnectTimer)
     this.#reconnectTimer = undefined
+    if (this.#gatewayProbeTimer !== undefined) clearTimeout(this.#gatewayProbeTimer)
+    this.#gatewayProbeTimer = undefined
     this.#setState("closed")
     for (const stream of [...this.#streams.values()]) stream.socket.destroy()
     const physical = this.#physical
@@ -446,6 +519,8 @@ export class ReverseGatewayConnection extends EventEmitter {
     this.#closing = true
     if (this.#reconnectTimer !== undefined) clearTimeout(this.#reconnectTimer)
     this.#reconnectTimer = undefined
+    if (this.#gatewayProbeTimer !== undefined) clearTimeout(this.#gatewayProbeTimer)
+    this.#gatewayProbeTimer = undefined
     this.#setState("closed")
     const physical = this.#physical
     physical?.pause()
@@ -658,7 +733,7 @@ export class ReverseGatewayConnection extends EventEmitter {
             const selection = this.#routeIdentityFields()
             if (detail.generation !== generation || ("gatewayId" in selection ? detail.gatewayId !== selection.gatewayId || Object.hasOwn(detail, "routeClass") : detail.routeClass !== selection.routeClass || Object.hasOwn(detail, "gatewayId"))) throw fatalError("GATEWAY_SELECTION_FAILED")
             if (typeof detail.resumed !== "boolean") throw fatalError("GATEWAY_SELECTION_FAILED")
-            if (detail.selectable === true) this.#gatewayUnavailableSince = undefined
+            if (detail.selectable === true) this.#markGatewayAvailable()
             else if (detail.selectable === false) this.#markGatewayUnavailable("GATEWAY_SELECT_UNAVAILABLE")
             selected = true
             if (detail.resumed) {
@@ -785,13 +860,7 @@ export class ReverseGatewayConnection extends EventEmitter {
       stream.sendWindow = Math.min(stream.sendWindow, detail.initialWindow)
       stream.state = stream.localEnded ? "half-closed" : "open"
       stream.opened = true
-      if (this.#gatewayUnavailableSince !== undefined) {
-        this.#gatewayUnavailableSince = undefined
-        logDiagnostic(this.#options.logger, "reverse_client.gateway_available", {
-          generation: this.#generation,
-          ...this.#routeIdentityFields()
-        })
-      }
+      this.#markGatewayAvailable()
       logDiagnostic(this.#options.logger, "virtual_stream.opened", {
         streamId: stream.id,
         generation: this.#generation,
@@ -1219,6 +1288,11 @@ export class ReverseGatewayConnection extends EventEmitter {
     this.#reconnectDelay = Math.min(this.#reconnectDelay * 2, this.#options.maxReconnectDelayMs ?? 30_000)
     this.#reconnectTimer = setTimeout(() => {
       this.#reconnectTimer = undefined
+      // clearTimeout() cannot stop a callback that is already queued, so a
+      // detach()/close() that lands in that window used to reconnect anyway:
+      // it reopened a physical transport after the caller had deliberately
+      // detached (observed as ENOENT reading a deleted recovery CA file).
+      if (this.#closing || this.#fatal) return
       if (this.#connectPromise === undefined) {
         this.#connectPromise = new Promise((resolve, reject) => { this.#resolveConnect = resolve; this.#rejectConnect = reject })
         this.#connectPromise.catch(() => {})
@@ -1336,6 +1410,9 @@ function validateOptions(options) {
   positiveInteger(options.maxConnectionBufferBytes, "maxConnectionBufferBytes", DEFAULT_CONNECTION_BUFFER_BYTES)
   if (options.spoolDirectory !== undefined && (typeof options.spoolDirectory !== "string" || options.spoolDirectory.length === 0)) throw new TypeError("spoolDirectory must be a non-empty string")
   positiveInteger(options.maxAgeMs, "maxAgeMs", 24 * 60 * 60 * 1000)
+  const gatewayProbeDelay = positiveInteger(options.gatewayProbeDelayMs, "gatewayProbeDelayMs", DEFAULT_GATEWAY_PROBE_DELAY_MS)
+  const maxGatewayProbeDelay = positiveInteger(options.maxGatewayProbeDelayMs, "maxGatewayProbeDelayMs", DEFAULT_MAX_GATEWAY_PROBE_DELAY_MS)
+  if (maxGatewayProbeDelay < gatewayProbeDelay) throw new RangeError("maxGatewayProbeDelayMs must not be smaller than gatewayProbeDelayMs")
   if (options.initialGeneration !== undefined && (!Number.isSafeInteger(options.initialGeneration) || options.initialGeneration < 0)) throw new RangeError("initialGeneration must be a non-negative safe integer")
 }
 

package/test/integration/multiplexed-drain.test.js

@@ -0,0 +1,179 @@
+/* eslint-disable jsdoc/escape-inline-tags, jsdoc/require-returns */
+import {describe, expect, it, withTestCleanup} from "@velocious/testing"
+import {createServer} from "node:net"
+
+import {StaticTokenStore} from "../../src/auth/static-token-store.js"
+import {MultiplexedSocketductConnection} from "../../src/client/multiplexed-connection.js"
+import {startMultiplexedTcpRelay} from "../../src/proxy/multiplexed-relay.js"
+
+/**
+ * `close()` terminates live virtual streams, which the relay answers with a
+ * stream reset — that is how in-flight work on the other side of a transport
+ * set died when the owning process shut down. `drain()` is the graceful verb:
+ * it refuses new streams, waits for the open ones, then releases the transport.
+ */
+
+/** @param {import('node:net').Server} server - Server. @returns {Promise<number>} */
+async function listen(server) {
+  await new Promise((resolve, reject) => {
+    server.once("error", reject)
+    server.listen(0, "127.0.0.1", () => {
+      server.off("error", reject)
+      resolve(undefined)
+    })
+  })
+  const address = server.address()
+
+  if (address === null || typeof address === "string") throw new Error("Expected a TCP address")
+
+  return address.port
+}
+
+/**
+ * @param {import('node:events').EventEmitter} emitter - Emitter.
+ * @param {string} event - Event.
+ * @returns {Promise<void>}
+ */
+async function onceEvent(emitter, event) {
+  await new Promise((resolve, reject) => {
+    emitter.once("error", reject)
+    emitter.once(event, () => {
+      emitter.off("error", reject)
+      resolve(undefined)
+    })
+  })
+}
+
+/**
+ * @param {Array<import("../../src/client/virtual-socket.js").VirtualSocket>} streams - Open streams.
+ * @param {number} index - Index.
+ * @returns {import("../../src/client/virtual-socket.js").VirtualSocket} - The stream.
+ */
+function streamAt(streams, index) {
+  const stream = streams[index]
+
+  if (stream === undefined) throw new Error(`Expected stream ${index} to be open`)
+
+  return stream
+}
+
+/** @returns {Promise<void>} */
+async function settleTurns() {
+  for (let index = 0; index < 10; index += 1) await new Promise((resolve) => setImmediate(resolve))
+}
+
+/**
+ * Relay plus one client connection with `streamCount` echoed streams open.
+ * @param {(cleanup: () => void | Promise<void>) => void} after - Cleanup registrar.
+ * @param {number} streamCount - Streams to open.
+ */
+async function openStreams(after, streamCount) {
+  /** @type {Set<import('node:net').Socket>} */
+  const targetSockets = new Set()
+  const target = createServer((socket) => {
+    targetSockets.add(socket)
+    socket.on("close", () => targetSockets.delete(socket))
+    socket.on("data", (chunk) => socket.write(chunk))
+  })
+  const targetPort = await listen(target)
+
+  after(() => {
+    for (const socket of targetSockets) socket.destroy()
+
+    return new Promise((resolve) => target.close(() => resolve(undefined)))
+  })
+
+  const relay = await startMultiplexedTcpRelay({
+    tokens: new StaticTokenStore([{username: "edge", token: "valid-token"}]),
+    authorizeTarget: ({target: requested}) => requested.host === "127.0.0.1" && requested.port === targetPort
+  })
+
+  after(() => relay.close())
+
+  const connection = new MultiplexedSocketductConnection({
+    relay: {host: "127.0.0.1", port: relay.port, token: "valid-token"}
+  })
+
+  after(() => connection.close())
+
+  /** @type {Array<import("../../src/client/virtual-socket.js").VirtualSocket>} */
+  const streams = []
+
+  for (let index = 0; index < streamCount; index += 1) {
+    streams.push(connection.createConnection({host: "127.0.0.1", port: targetPort, sessionName: `drain-${index}`}))
+  }
+
+  await Promise.all(streams.map((stream) => onceEvent(stream, "connect")))
+  expect(connection.status().virtualStreams).toBe(streamCount)
+
+  return {connection, relay, streams, targetPort}
+}
+
+describe("multiplexed connection drain", () => {
+  it("drain() keeps in-flight virtual streams alive until they finish", async () => withTestCleanup(async (after) => {
+    const {connection, relay, streams} = await openStreams(after, 2)
+    const draining = connection.drain()
+
+    await settleTurns()
+
+    expect(connection.status().virtualStreams).toBe(2)
+    expect(streamAt(streams, 0).destroyed).toBeFalse()
+    expect(streamAt(streams, 1).destroyed).toBeFalse()
+    expect(relay.status().authenticatedConnections).toBe(1)
+
+    for (const stream of streams) stream.destroy()
+    await draining
+
+    expect(connection.status().virtualStreams).toBe(0)
+    expect(relay.status().authenticatedConnections).toBe(0)
+  }))
+
+  it("drain() keeps working streams usable instead of resetting them", async () => withTestCleanup(async (after) => {
+    const {connection, streams} = await openStreams(after, 1)
+    const draining = connection.drain()
+
+    await settleTurns()
+
+    streamAt(streams, 0).write("still-works")
+
+    /** @type {string} */
+    let echoed = ""
+
+    await new Promise((resolve, reject) => {
+      streamAt(streams, 0).once("error", reject)
+      streamAt(streams, 0).on("data", (chunk) => {
+        echoed += chunk.toString()
+        if (echoed.length >= 11) resolve(undefined)
+      })
+    })
+
+    expect(echoed).toBe("still-works")
+
+    streamAt(streams, 0).destroy()
+    await draining
+  }))
+
+  it("drain() rejects new virtual streams while draining", async () => withTestCleanup(async (after) => {
+    const {connection, streams, targetPort} = await openStreams(after, 1)
+    const draining = connection.drain()
+
+    await settleTurns()
+
+    expect(() => connection.createConnection({host: "127.0.0.1", port: targetPort, sessionName: "late"})).toThrow(/connection is closed/i)
+    expect(connection.status().virtualStreams).toBe(1)
+
+    streamAt(streams, 0).destroy()
+    await draining
+  }))
+
+  it("close() still terminates in-flight virtual streams immediately", async () => withTestCleanup(async (after) => {
+    const {connection, relay, streams} = await openStreams(after, 2)
+
+    await connection.close()
+
+    expect(connection.status().virtualStreams).toBe(0)
+    expect(streamAt(streams, 0).destroyed).toBeTrue()
+    expect(streamAt(streams, 1).destroyed).toBeTrue()
+    expect(relay.status().authenticatedConnections).toBe(0)
+  }))
+})

package/test/integration/reverse-route-class.test.js

@@ -56,6 +56,15 @@ async function waitUntil(predicate, label) {
   }
 }
 
+/** @param {() => boolean} predicate - Completion predicate. @param {string} label - Failure label. @param {number} timeoutMs - Deadline. */
+async function waitUntilWithin(predicate, label, timeoutMs) {
+  const deadline = Date.now() + timeoutMs
+  while (!predicate()) {
+    if (Date.now() >= deadline) throw new Error(`Timed out: ${label}`)
+    await new Promise((resolve) => setTimeout(resolve, 10))
+  }
+}
+
 /** @param {string} prefix - Echo response prefix. */
 async function createEchoTarget(prefix) {
   const sockets = new Set()
@@ -470,4 +479,74 @@ describe("reverse gateway route-class", () => {
     expect(await exchange(stream, "probe")).toBe("a:probe")
     stream.destroy()
   }))
+
+  it("a route class registered down stays down, keeps being reached, and rejoins automatically", async () => withTestCleanup(async (after) => {
+    const root = await mkdtemp(join(tmpdir(), "socketduct-route-class-down-"))
+    after(() => rm(root, {recursive: true, force: true}))
+    const certificates = generateTestCertificates(root)
+    const targetA = await createEchoTarget("a:")
+    after(() => targetA.close())
+
+    const hub = await startReverseGatewayHub({
+      tokens: new StaticTokenStore([
+        {username: "home-a", token: "gateway-a-token"},
+        {username: "tensorbuzz", token: "client-token"}
+      ]),
+      tls: {certFile: certificates.serverCertFile, keyFile: certificates.serverKeyFile},
+      knownGatewayIds: ["server-a-starlink"],
+      knownRouteClasses: ["starlink"],
+      authorizeGateway: ({principal, gatewayId}) => principal.username === "home-a" && gatewayId === "server-a-starlink",
+      authorizeClientGateway: () => false,
+      authorizeClientRouteClass: ({principal, routeClass}) => principal.username === "tensorbuzz" && routeClass === "starlink",
+      resolveRouteClassGateways: () => ["server-a-starlink"],
+      resolveOpenGateways: ({routeClass, target}) => (routeClass === "starlink" && target.host === targetA.host ? ["server-a-starlink"] : []),
+      authorizeOpen: ({gatewayId, target}) => gatewayId === "server-a-starlink" && target.port === targetA.port
+    })
+    after(() => hub.close())
+    const tls = {caFile: certificates.caCertFile, servername: "localhost"}
+
+    const client = new ReverseGatewayConnection({
+      routeClass: "starlink",
+      connectionId: "route_class_down_client",
+      hub: {host: "127.0.0.1", port: hub.port, token: "client-token"},
+      tls,
+      gatewayProbeDelayMs: 20,
+      maxGatewayProbeDelayMs: 40,
+      reconnectDelayMs: 10,
+      maxReconnectDelayMs: 20
+    })
+    after(() => client.close())
+
+    await client.connect()
+    expect(client.status().gatewayUnavailable).toBe(true)
+
+    // The member keeps reaching the hub while its route class is down.
+    await waitUntilWithin(() => client.status().gatewayProbeAttempts >= 3, "gateway re-reaching", 4_000)
+
+    // A down registration never expires on its own: the previous five-second
+    // mark would have silently returned this member to service with no gateway.
+    await new Promise((resolve) => setTimeout(resolve, 5_100))
+    expect(client.status().gatewayUnavailable).toBe(true)
+    expect(client.status().gatewayProbeAttempts >= 3).toBe(true)
+
+    const gatewayA = new OutboundReverseGateway({
+      gatewayId: "server-a-starlink",
+      connectionId: "down_gateway_a",
+      hub: {host: "127.0.0.1", port: hub.port, token: "gateway-a-token"},
+      tls,
+      authorizeTarget: (target) => target.host === targetA.host && target.port === targetA.port,
+      reconnectDelayMs: 10,
+      maxReconnectDelayMs: 20
+    })
+    after(() => gatewayA.close())
+    await gatewayA.start()
+    await waitUntil(() => hub.status().healthyGateways === 1, "gateway A registration")
+
+    // The same member registers the class up again and is taken back into use.
+    await waitUntilWithin(() => client.status().gatewayUnavailable === false, "automatic re-registration", 4_000)
+    const stream = client.createConnection({host: targetA.host, port: targetA.port})
+    await onceEvent(stream, "connect")
+    expect(await exchange(stream, "probe")).toBe("a:probe")
+    stream.destroy()
+  }))
 })

package/test/unit/client/multiplexed-transport-set.test.js

@@ -58,6 +58,7 @@ class FakeMember extends EventEmitter {
     this.closeError = closeError
     this.connectCalls = 0
     this.closeCalls = 0
+    this.drainCalls = 0
     this.opens = 0
     /** @type {FakeSocket[]} */
     this.sockets = []
@@ -161,6 +162,11 @@ class FakeMember extends EventEmitter {
     this.setState("closed")
   }
 
+  async drain() {
+    this.drainCalls += 1
+    this.setState("closed")
+  }
+
   /** @param {string} state - Physical state. @param {Error} [error] - Terminal error. */
   setState(state, error) {
     this.state = state
@@ -552,16 +558,30 @@ describe("multiplexed transport set", () => {
     await set.close()
   })
 
-  it("a stale gateway-unavailable mark cannot wedge the set when every ready member is marked", async () => {
+  it("a member the hub registered down is never selected, even when every member is down", async () => {
     const {set, members} = fixture()
     const started = set.start()
     members.forEach((member) => member.ready())
     await started
     members.forEach((member) => { member.gatewayUnavailable = true })
+    let callbacks = 0
     const socket = set.createConnection({host: "target.internal", port: 443}, (error) => {
       expect(error).toBe(null)
+      callbacks += 1
     })
+    await new Promise((resolve) => setImmediate(resolve))
+    expect(callbacks).toBe(0)
+    expect(socket.gatewayId).toBe(undefined)
+    expect(set.status().pendingOpens).toBe(1)
+    expect(members.map((member) => member.opens)).toEqual([0, 0, 0])
+    expect(set.status().members.map((member) => member.gatewayUnavailable)).toEqual([true, true, true])
+
+    at(members, 2).gatewayUnavailable = false
+    at(members, 2).setState("ready")
+    await new Promise((resolve) => setImmediate(resolve))
+    expect(callbacks).toBe(1)
     expect(socket.gatewayId !== undefined).toBe(true)
+    expect(members.map((member) => member.opens)).toEqual([0, 0, 1])
     expect(set.status().pendingOpens).toBe(0)
     await set.close()
   })