socketduct 0.1.15

package/test/unit/client/multiplexed-transport-set.test.js

@@ -0,0 +1,906 @@
+/* eslint-disable jsdoc/escape-inline-tags, jsdoc/no-undefined-types, jsdoc/require-param, jsdoc/require-returns */
+import assert from "node:assert/strict"
+import {EventEmitter} from "node:events"
+import {join} from "node:path"
+import {Duplex} from "node:stream"
+import test from "node:test"
+
+import {
+  MultiplexedSocketductTransportSet,
+  SocketductTransportSetLimitError,
+  SocketductTransportSetTerminalError
+} from "../../../src/client/multiplexed-transport-set.js"
+
+class FakeSocket extends Duplex {
+  connecting = true
+  pending = true
+
+  /** @type {Buffer[]} */
+  writes = []
+
+  /** @override */
+  _read() {}
+
+  /**
+   * @override
+   * @param {Buffer} chunk - Bytes.
+   * @param {BufferEncoding} _encoding - Encoding.
+   * @param {(error?: Error | null) => void} callback - Completion.
+   */
+  _write(chunk, _encoding, callback) {
+    this.writes.push(Buffer.from(chunk))
+    callback()
+  }
+
+  ready() {
+    this.connecting = false
+    this.pending = false
+    this.emit("connect")
+    this.emit("ready")
+  }
+
+  address() { return {address: "127.0.0.1", family: "IPv4", port: 31_000} }
+
+  ref() { return this }
+
+  unref() { return this }
+}
+
+class FakeMember extends EventEmitter {
+  /**
+   * @param {import('../../../src/client/multiplexed-connection.js').MultiplexedConnectionOptions | import('../../../src/reverse/client.js').ReverseGatewayConnectionOptions} options - Member options.
+   * @param {string} gatewayId - Gateway ID.
+   * @param {Error | undefined} closeError - Injected close failure.
+   */
+  constructor(options, gatewayId, closeError) {
+    super()
+    this.options = options
+    this.gatewayId = gatewayId
+    this.closeError = closeError
+    this.connectCalls = 0
+    this.closeCalls = 0
+    this.opens = 0
+    /** @type {FakeSocket[]} */
+    this.sockets = []
+    /** @type {Record<string, unknown>[]} */
+    this.createConnectionCalls = []
+    /** @type {Record<string, unknown>[]} */
+    this.recoveryConnectionCalls = []
+    this.state = "configured"
+    this.generation = 0
+    this.physicalConnects = 0
+    /** @type {Promise<void>} */
+    this.connectPromise = new Promise((resolve, reject) => {
+      this.resolveConnect = resolve
+      this.rejectConnect = reject
+    })
+    this.connectPromise.catch(() => {})
+  }
+
+  connect() {
+    this.connectCalls += 1
+    this.setState("connecting")
+    return this.connectPromise
+  }
+
+  ready() {
+    this.generation += 1
+    this.physicalConnects += 1
+    this.setState("ready")
+    this.resolveConnect()
+  }
+
+  /** @param {Error} [error] - Transient connection error. */
+  degrade(error) {
+    this.setState("reconnecting", error)
+  }
+
+  /** @param {Error} error - Terminal connection failure. */
+  fail(error) {
+    this.setState("closed", error)
+    this.rejectConnect(error)
+  }
+
+  /** @param {Record<string, unknown>} options - Open options. @param {(error: Error | null, socket: import('../../../src/client/virtual-socket.js').VirtualSocket) => void} [callback] - Callback. */
+  createConnection(options, callback) {
+    this.opens += 1
+    this.createConnectionCalls.push(options)
+    const socket = new FakeSocket()
+    this.sockets.push(socket)
+    Object.assign(socket, {options})
+    if (callback !== undefined) socket.once("connect", () => callback(null, /** @type {import('../../../src/client/virtual-socket.js').VirtualSocket} */ (/** @type {unknown} */ (socket))))
+    queueMicrotask(() => socket.ready())
+    return /** @type {import('../../../src/client/virtual-socket.js').VirtualSocket} */ (/** @type {unknown} */ (socket))
+  }
+
+  /** @param {Record<string, unknown>} options - Recovery options. @param {(error: Error | null, socket: import('../../../src/client/virtual-socket.js').VirtualSocket) => void} [callback] - Callback. */
+  registerRecoveredConnection(options, callback) {
+    this.opens += 1
+    this.recoveryConnectionCalls.push(options)
+    const socket = new FakeSocket()
+    this.sockets.push(socket)
+    Object.assign(socket, {options})
+    if (callback !== undefined) socket.once("connect", () => callback(null, /** @type {import('../../../src/client/virtual-socket.js').VirtualSocket} */ (/** @type {unknown} */ (socket))))
+    queueMicrotask(() => socket.ready())
+    return /** @type {import('../../../src/client/virtual-socket.js').VirtualSocket} */ (/** @type {unknown} */ (socket))
+  }
+
+  status() {
+    return {
+      physicalState: this.state,
+      connectionGeneration: this.generation,
+      physicalConnects: this.physicalConnects,
+      reconnectAttempts: this.state === "reconnecting" ? 1 : 0,
+      reconnectSuccesses: 0,
+      virtualStreams: this.opens,
+      queuedBytes: 0,
+      dataQueuedBytes: 0,
+      controlQueuedBytes: 0,
+      spoolBytes: 0,
+      spoolCleanupFailures: 0
+    }
+  }
+
+  async close() {
+    this.closeCalls += 1
+    this.setState("closed")
+    this.rejectConnect(Object.assign(new Error("member closed"), {code: "SOCKETDUCT_TRANSPORT_SET_CLOSED"}))
+    if (this.closeError !== undefined) throw this.closeError
+  }
+
+  async detach() {
+    this.closeCalls += 1
+    this.setState("closed")
+  }
+
+  /** @param {string} state - Physical state. @param {Error} [error] - Terminal error. */
+  setState(state, error) {
+    this.state = state
+    this.emit("stateChange", state, error)
+  }
+}
+
+/** @param {Partial<import('../../../src/client/multiplexed-transport-set.js').TransportSetOptions>} [overrides] - Overrides. */
+function options(overrides = {}) {
+  return /** @type {import('../../../src/client/multiplexed-transport-set.js').TransportSetOptions} */ ({
+    connectionId: "process_identity_01",
+    spoolDirectory: "/tmp/socketduct-transport-set-test",
+    maxTotalBufferBytes: 1_024,
+    maxTotalSpoolBytes: 1_024,
+    maxTotalQueuedBytes: 2_048,
+    gateways: [
+      {gatewayId: "starlink", endpoint: {relay: {host: "127.0.0.1", port: 3101, token: "secret"}}},
+      {gatewayId: "zyxel", endpoint: {relay: {host: "127.0.0.1", port: 3102, token: "secret"}}},
+      {gatewayId: "o2gateway", endpoint: {relay: {host: "127.0.0.1", port: 3103, token: "secret"}}}
+    ],
+    ...overrides
+  })
+}
+
+/**
+ * @param {Partial<import('../../../src/client/multiplexed-transport-set.js').TransportSetOptions>} [overrides] - Overrides.
+ * @param {Record<string, Error>} [closeErrors] - Close errors by gateway.
+ */
+function fixture(overrides = {}, closeErrors = {}) {
+  /** @type {FakeMember[]} */
+  const members = []
+  const set = new MultiplexedSocketductTransportSet(options(overrides), {
+    createMember(memberOptions, gatewayId) {
+      const member = new FakeMember(memberOptions, gatewayId, closeErrors[gatewayId])
+      members.push(member)
+      return member
+    }
+  })
+  return {set, members}
+}
+
+/** @param {unknown} socket - Fake assigned socket. @returns {string} */
+function gatewayOf(socket) {
+  return /** @type {{gatewayId: string}} */ (socket).gatewayId
+}
+
+/** @template T @param {T[]} values - Values. @param {number} index - Index. @returns {T} */
+function at(values, index) {
+  const value = values[index]
+  assert(value !== undefined)
+  return value
+}
+
+/** @param {EventEmitter} emitter - Source. @param {string} event - Event. @returns {Promise<unknown[]>} */
+function onceEvent(emitter, event) {
+  return new Promise((resolve) => emitter.once(event, (...args) => resolve(args)))
+}
+
+/** @param {FakeMember[]} members - Fake members. @returns {Array<[string, string]>} */
+function memberPairs(members) {
+  return members.map((member) => {
+    const {connectionId, spoolDirectory} = /** @type {import('../../../src/client/multiplexed-connection.js').MultiplexedConnectionOptions} */ (member.options)
+    assert(typeof connectionId === "string")
+    assert(typeof spoolDirectory === "string")
+    return [connectionId, spoolDirectory]
+  })
+}
+
+test("validates stable gateway identities, pinned endpoints, and process limits", () => {
+  for (const gatewayId of ["", " ../wan ", "wan/../../escape", "a", "x".repeat(65)]) {
+    assert.throws(() => new MultiplexedSocketductTransportSet(options({
+      gateways: [{gatewayId, endpoint: {relay: {host: "127.0.0.1", port: 3101, token: "secret"}}}]
+    })), /gatewayId/i)
+  }
+  assert.throws(() => new MultiplexedSocketductTransportSet(options({
+    gateways: [
+      {gatewayId: "duplicate", endpoint: {relay: {host: "127.0.0.1", port: 3101, token: "secret"}}},
+      {gatewayId: "duplicate", endpoint: {relay: {host: "127.0.0.1", port: 3102, token: "secret"}}}
+    ]
+  })), /duplicate gatewayId/i)
+  assert.throws(() => new MultiplexedSocketductTransportSet(options({
+    // @ts-expect-error Deliberately malformed entry must fail at runtime.
+    gateways: [{gatewayId: "starlink", relay: {host: "127.0.0.1", port: 3101, token: "secret"}}]
+  })), /endpoint/i)
+  assert.throws(() => new MultiplexedSocketductTransportSet(options({
+    // @ts-expect-error Deliberately malformed port must fail at runtime.
+    gateways: [{gatewayId: "starlink", endpoint: {relay: {host: "127.0.0.1", port: "3101", token: "secret"}}}]
+  })), /host, port, and token/i)
+  assert.throws(() => new MultiplexedSocketductTransportSet(options({
+    gateways: [{
+      gatewayId: "starlink",
+      // @ts-expect-error Misspelled pinning factory must fail instead of being ignored.
+      endpoint: {relay: {host: "127.0.0.1", port: 3101, token: "secret"}, physicalSocketFactory: () => {}}
+    }]
+  })), /unknown endpoint field/i)
+  assert.throws(() => new MultiplexedSocketductTransportSet(options({
+    gateways: [
+      {gatewayId: "starlink", endpoint: {relay: {host: "relay.example", port: 3101, token: "secret"}}},
+      {gatewayId: "zyxel", endpoint: {relay: {host: "relay.example", port: 3101, token: "secret"}}}
+    ]
+  })), /same unpinned endpoint/i)
+  assert.throws(() => new MultiplexedSocketductTransportSet(options({minimumReadyMembers: 4})), /minimumReadyMembers/i)
+  for (const name of ["maxTotalBufferBytes", "maxTotalSpoolBytes", "maxTotalQueuedBytes", "maxPendingOpens"]) {
+    assert.throws(() => new MultiplexedSocketductTransportSet(options({[name]: "1024"})), new RegExp(name))
+  }
+})
+
+test("logs transport-set startup, generation replacement, and terminal failure without error text", async () => {
+  /** @type {Array<{event: string} & Record<string, unknown>>} */
+  const entries = []
+  const logger = {log: (/** @type {string} */ event, /** @type {Record<string, unknown>} */ fields = {}) => entries.push({event, ...fields})}
+  const {set, members} = fixture({logger})
+  assert(members.every((member) => member.options.logger === logger))
+
+  const started = set.start()
+  at(members, 0).ready()
+  await started
+  at(members, 0).degrade(Object.assign(new Error("alphabeticsecret eyJhbGciOiJIUzI1NiJ9.private.signature"), {code: "ECONNRESET"}))
+  at(members, 0).ready()
+  at(members, 0).fail(Object.assign(new Error("first lowentropysecret"), {code: "GATEWAY_DISCONNECTED"}))
+  at(members, 1).fail(new Error("second alphabeticsecret"))
+  at(members, 2).fail(new Error("third eyJhbGciOiJIUzI1NiJ9.private.signature"))
+
+  assert(entries.some(({event}) => event === "transport_set.started"))
+  assert(entries.some(({event, gatewayId, generation}) => event === "transport_set.member_state" && gatewayId === "starlink" && generation === 1))
+  assert(entries.some(({event, gatewayId, generation}) => event === "transport_set.member_replaced" && gatewayId === "starlink" && generation === 2))
+  assert(entries.some(({event}) => event === "transport_set.terminal_failure"))
+  const serialized = JSON.stringify(entries)
+  assert.doesNotMatch(serialized, /alphabeticsecret|lowentropysecret|eyJhbGci|127\.0\.0\.1|secret/)
+  await set.close()
+})
+
+test("derives stable isolated connection identities and spool namespaces per gateway", () => {
+  const first = fixture()
+  const second = fixture()
+  const firstPairs = memberPairs(first.members)
+  const secondPairs = memberPairs(second.members)
+  assert.deepEqual(firstPairs, secondPairs)
+  assert.equal(new Set(firstPairs.map(([connectionId]) => connectionId)).size, 3)
+  assert.equal(new Set(firstPairs.map(([, spoolDirectory]) => spoolDirectory)).size, 3)
+  const baseSpoolDirectory = options().spoolDirectory
+  assert(typeof baseSpoolDirectory === "string")
+  for (const [connectionId, spoolDirectory] of firstPairs) {
+    assert.match(connectionId, /^[A-Za-z0-9_-]{8,128}$/)
+    assert.equal(spoolDirectory.startsWith(join(baseSpoolDirectory, "gateway-")), true)
+    assert.equal(spoolDirectory.includes(".."), false)
+  }
+})
+
+test("warms every configured member concurrently and coalesces start at minimum readiness", async () => {
+  const {set, members} = fixture({minimumReadyMembers: 1})
+  const firstStart = set.start()
+  const secondStart = set.start()
+  assert.equal(firstStart, secondStart)
+  assert.deepEqual(members.map((member) => member.connectCalls), [1, 1, 1])
+  assert.equal(set.status().connectingMembers, 3)
+
+  at(members, 1).ready()
+  await firstStart
+  assert.equal(set.status().readyMembers, 1)
+  assert.equal(set.status().state, "degraded")
+  assert.deepEqual(members.map((member) => member.connectCalls), [1, 1, 1])
+
+  at(members, 0).ready()
+  at(members, 2).ready()
+  assert.equal(set.status().readyMembers, 3)
+  assert.equal(set.status().state, "ready")
+  await set.close()
+})
+
+test("start followed immediately by createConnection returns a socket and waits for readiness", async () => {
+  const {set, members} = fixture()
+  const started = set.start()
+  const socket = set.createConnection({host: "target.internal", port: 443})
+  assert.equal(socket.connecting, true)
+  assert.equal(socket.gatewayId, undefined)
+  assert.equal(set.status().pendingOpens, 1)
+  assert.deepEqual(members.map((member) => member.opens), [0, 0, 0])
+
+  at(members, 0).ready()
+  await started
+  assert.equal(gatewayOf(socket), "starlink")
+  assert.deepEqual(members.map((member) => member.opens), [1, 0, 0])
+  await set.close()
+})
+
+test("createConnection before start lazily warms the set and waits for minimum readiness", async () => {
+  const {set, members} = fixture({minimumReadyMembers: 2})
+  const socket = set.createConnection({host: "target.internal", port: 443})
+  assert.deepEqual(members.map((member) => member.connectCalls), [1, 1, 1])
+
+  at(members, 0).ready()
+  assert.deepEqual(members.map((member) => member.opens), [0, 0, 0])
+  at(members, 1).ready()
+  assert.equal(gatewayOf(socket), "starlink")
+  assert.deepEqual(members.map((member) => member.opens), [1, 0, 0])
+  await set.close()
+})
+
+test("pending opens snapshot caller options and nested target at call time", async () => {
+  const {set, members} = fixture()
+  const started = set.start()
+  const target = {host: "original.internal", port: 443}
+  const openOptions = {
+    host: "original-fallback.internal",
+    port: 80,
+    target,
+    sessionName: "original-session",
+    identifier: "snapshot_stream_01",
+    allowHalfOpen: true,
+    readableHighWaterMark: 7,
+    writableHighWaterMark: 9
+  }
+  const socket = set.createConnection(openOptions)
+
+  openOptions.host = "mutated-fallback.internal"
+  openOptions.port = 81
+  openOptions.sessionName = "mutated-session"
+  openOptions.identifier = "mutated_stream_01"
+  openOptions.allowHalfOpen = false
+  openOptions.readableHighWaterMark = 70
+  openOptions.writableHighWaterMark = 90
+  target.host = "mutated.internal"
+  target.port = 444
+
+  at(members, 0).ready()
+  await started
+  const accepted = at(at(members, 0).createConnectionCalls, 0)
+  assert.deepEqual(accepted, {
+    host: "original-fallback.internal",
+    port: 80,
+    target: {host: "original.internal", port: 443},
+    sessionName: "original-session",
+    identifier: "snapshot_stream_01",
+    allowHalfOpen: true,
+    readableHighWaterMark: 7,
+    writableHighWaterMark: 9
+  })
+  assert.notEqual(accepted, openOptions)
+  assert.notEqual(accepted.target, target)
+  assert.equal(socket.remoteAddress, "original.internal")
+  assert.equal(socket.remotePort, 443)
+  socket.destroy()
+  await set.close()
+})
+
+test("pending opens preserve null target fallback to host and port", async () => {
+  const {set, members} = fixture()
+  const started = set.start()
+  const options = /** @type {import('../../../src/client/multiplexed-connection.js').VirtualConnectionOptions} */ (/** @type {unknown} */ ({
+    target: null,
+    host: "fallback.internal",
+    port: 443,
+    sessionName: "null-target-fallback"
+  }))
+  const socket = set.createConnection(options)
+  assert.equal(socket.remoteAddress, "fallback.internal")
+  assert.equal(socket.remotePort, 443)
+
+  at(members, 0).ready()
+  await started
+  assert.deepEqual(at(at(members, 0).createConnectionCalls, 0), {
+    target: null,
+    host: "fallback.internal",
+    port: 443,
+    sessionName: "null-target-fallback"
+  })
+  socket.destroy()
+  await set.close()
+})
+
+test("round-robins each virtual open fairly across currently ready sticky members", async () => {
+  const {set, members} = fixture()
+  const started = set.start()
+  members.forEach((member) => member.ready())
+  await started
+
+  const firstThirty = Array.from({length: 30}, () => set.createConnection({host: "target.internal", port: 443}))
+  assert.deepEqual(firstThirty.map(gatewayOf).slice(0, 6), ["starlink", "zyxel", "o2gateway", "starlink", "zyxel", "o2gateway"])
+  assert.deepEqual(set.status().members.map((member) => member.assignments), [10, 10, 10])
+
+  at(members, 1).degrade()
+  const whileDown = Array.from({length: 25}, () => set.createConnection({host: "target.internal", port: 443}))
+  assert.equal(whileDown.every((socket) => gatewayOf(socket) !== "zyxel"), true)
+  const siblingAssignments = set.status().members.filter((member) => member.gatewayId !== "zyxel").map((member) => member.assignments - 10)
+  assert(Math.max(...siblingAssignments) - Math.min(...siblingAssignments) <= 1)
+  assert.equal(gatewayOf(at(firstThirty, 1)), "zyxel")
+
+  at(members, 1).ready()
+  const beforeRecovery = set.status().members.map((member) => member.assignments)
+  const afterRecovery = Array.from({length: 100}, () => set.createConnection({host: "target.internal", port: 443}))
+  assert.equal(afterRecovery.some((socket) => gatewayOf(socket) === "zyxel"), true)
+  const recoveryAssignments = set.status().members.map((member, index) => member.assignments - at(beforeRecovery, index))
+  assert(Math.max(...recoveryAssignments) - Math.min(...recoveryAssignments) <= 1)
+  assert.equal(set.status().virtualOpens, 155)
+  await set.close()
+})
+
+test("25 and 100 continuously healthy opens have exact deterministic distributions", async () => {
+  for (const [count, expected] of /** @type {Array<[number, number[]]>} */ ([[25, [9, 8, 8]], [100, [34, 33, 33]]])) {
+    const {set, members} = fixture()
+    const started = set.start()
+    members.forEach((member) => member.ready())
+    await started
+    Array.from({length: count}, () => set.createConnection({host: "target.internal", port: 443}))
+    assert.deepEqual(set.status().members.map((member) => member.assignments), expected)
+    assert.deepEqual(members.map((member) => member.opens), expected)
+    await set.close()
+  }
+})
+
+test("all-members reconnecting queues FIFO opens until one ready member recovers", async () => {
+  const {set, members} = fixture()
+  const started = set.start()
+  members.forEach((member) => member.ready())
+  await started
+  members.forEach((member) => member.degrade())
+  /** @type {string[]} */
+  const callbackOrder = []
+  const sockets = ["first", "second", "third"].map((sessionName) => set.createConnection({host: "target.internal", port: 443, sessionName}, (error) => {
+    assert.equal(error, null)
+    callbackOrder.push(sessionName)
+  }))
+  assert.equal(sockets.every((socket) => socket.gatewayId === undefined), true)
+  assert.equal(set.status().pendingOpens, 3)
+  assert.equal(set.status().virtualOpens, 0)
+  assert.deepEqual(members.map((member) => member.opens), [0, 0, 0])
+
+  at(members, 1).ready()
+  await new Promise((resolve) => setImmediate(resolve))
+  assert.deepEqual(callbackOrder, ["first", "second", "third"])
+  assert.equal(sockets.every((socket) => socket.gatewayId === "zyxel"), true)
+  assert.deepEqual(at(members, 1).createConnectionCalls.map((call) => call.sessionName), ["first", "second", "third"])
+  assert.equal(set.status().pendingOpens, 0)
+  assert.equal(set.status().virtualOpens, 3)
+  await set.close()
+})
+
+test("transient reverse-hub codes keep opens pending until a member reconnects", async () => {
+  const {set, members} = fixture()
+  const started = set.start()
+  members.forEach((member) => member.ready())
+  await started
+  const transientCodes = ["GATEWAY_UNAVAILABLE", "GATEWAY_DISCONNECTED", "GATEWAY_REPLACED", "HUB_SHUTDOWN"]
+  members.forEach((member, index) => member.degrade(Object.assign(new Error("transient reverse route"), {code: at(transientCodes, index)})))
+  at(members, 0).setState("reconnecting", Object.assign(new Error("transient hub shutdown"), {code: at(transientCodes, 3)}))
+  let callbacks = 0
+  const socket = set.createConnection({host: "target.internal", port: 443}, (error) => {
+    assert.equal(error, null)
+    callbacks += 1
+  })
+  const ready = onceEvent(socket, "ready")
+  await new Promise((resolve) => setImmediate(resolve))
+  assert.equal(callbacks, 0)
+  assert.equal(set.status().state, "degraded")
+  assert.equal(set.status().pendingOpens, 1)
+  assert.deepEqual(set.status().members.map((member) => member.terminalErrorCode), [null, null, null])
+  at(members, 1).ready()
+  await ready
+  assert.equal(callbacks, 1)
+  assert.equal(socket.gatewayId, "zyxel")
+  await set.close()
+})
+
+test("pending assignment preserves callback, connect, ready, and immutable gateway ordering", async () => {
+  const {set, members} = fixture()
+  const started = set.start()
+  /** @type {string[]} */
+  const events = []
+  let callbacks = 0
+  const socket = set.createConnection({host: "target.internal", port: 443}, (error, callbackSocket) => {
+    callbacks += 1
+    assert.equal(error, null)
+    assert.equal(callbackSocket, socket)
+    events.push("callback")
+  })
+  socket.on("connect", () => events.push("connect"))
+  socket.on("ready", () => events.push("ready"))
+  const ready = onceEvent(socket, "ready")
+  at(members, 0).ready()
+  await started
+  await ready
+  assert.deepEqual(events, ["callback", "connect", "ready"])
+  assert.equal(callbacks, 1)
+  assert.equal(socket.gatewayId, "starlink")
+  const descriptor = Object.getOwnPropertyDescriptor(socket, "gatewayId")
+  assert.equal(descriptor?.set, undefined)
+  assert.equal(descriptor?.configurable, false)
+  at(members, 0).degrade()
+  at(members, 1).ready()
+  assert.equal(socket.gatewayId, "starlink")
+  await set.close()
+})
+
+test("destroy before assignment cancels the pending open and cannot leak a later member stream", async () => {
+  const {set, members} = fixture()
+  const started = set.start()
+  const socket = set.createConnection({host: "target.internal", port: 443})
+  const closed = onceEvent(socket, "close")
+  socket.destroy()
+  await closed
+  assert.equal(set.status().pendingOpens, 0)
+  at(members, 0).ready()
+  await started
+  assert.deepEqual(members.map((member) => member.opens), [0, 0, 0])
+  await set.close()
+})
+
+test("terminal startup failure reaches start and each pending socket exactly once", async () => {
+  const {set, members} = fixture()
+  const start = set.start()
+  let callbacks = 0
+  /** @type {import('../../../src/client/virtual-socket.js').VirtualSocket | undefined} */
+  let pendingSocket
+  const callbackResult = new Promise((resolve) => {
+    pendingSocket = set.createConnection({host: "target.internal", port: 443}, (error, socket) => {
+      callbacks += 1
+      resolve({error, socket})
+    })
+  })
+  assert(pendingSocket !== undefined)
+  const closed = onceEvent(pendingSocket, "close")
+  const failures = members.map((member) => new Error(`${member.gatewayId} unavailable`))
+  members.forEach((member, index) => member.fail(at(failures, index)))
+  await assert.rejects(start, (error) => error instanceof AggregateError && error.errors.length === 3)
+  const result = /** @type {{error: Error, socket: import('../../../src/client/virtual-socket.js').VirtualSocket}} */ (await callbackResult)
+  await closed
+  assert(result.error instanceof AggregateError)
+  assert.equal(callbacks, 1)
+  assert.equal(set.status().state, "failed")
+  assert.equal(set.status().pendingOpens, 0)
+  assert.deepEqual(members.map((member) => member.opens), [0, 0, 0])
+  await set.close()
+})
+
+test("terminal loss preserves ordered member causes for pending and future opens", async () => {
+  const {set, members} = fixture()
+  const started = set.start()
+  members.forEach((member) => member.ready())
+  await started
+  members.forEach((member) => member.degrade())
+
+  /** @type {Error | undefined} */
+  let pendingError
+  const pendingSocket = set.createConnection({host: "target.internal", port: 443}, (error) => { pendingError = error ?? undefined })
+  const pendingClosed = onceEvent(pendingSocket, "close")
+  const causes = [
+    Object.assign(new Error("starlink certificate rejected"), {code: "ERR_TLS_CERT_ALTNAME_INVALID"}),
+    Object.assign(new Error("zyxel authentication rejected"), {code: "AUTH_REQUIRED"}),
+    Object.assign(new Error("o2 protocol rejected"), {code: "PROTOCOL_MISMATCH"})
+  ]
+  members.forEach((member, index) => member.fail(at(causes, index)))
+  await pendingClosed
+  assert(pendingError instanceof SocketductTransportSetTerminalError)
+  assert.equal(/** @type {{code?: string}} */ (pendingError).code, "SOCKETDUCT_TRANSPORT_SET_TERMINAL")
+  assert.deepEqual(pendingError.errors, causes)
+  const status = set.status()
+  assert.deepEqual(status.members.map((member) => member.terminalErrorCode), causes.map((error) => error.code))
+  assert.doesNotMatch(JSON.stringify(status), /certificate rejected|authentication rejected|protocol rejected/)
+  assert.equal(status.state, "failed")
+
+  let callbacks = 0
+  /** @type {import('../../../src/client/virtual-socket.js').VirtualSocket | undefined} */
+  let failedSocket
+  const callbackResult = new Promise((resolve) => {
+    failedSocket = set.createConnection({host: "target.internal", port: 443}, (error, socket) => {
+      callbacks += 1
+      resolve({error, socket})
+    })
+  })
+  assert(failedSocket !== undefined)
+  const closed = onceEvent(failedSocket, "close")
+  const result = /** @type {{error: Error, socket: import('../../../src/client/virtual-socket.js').VirtualSocket}} */ (await callbackResult)
+  await closed
+  assert.equal(result.error, pendingError)
+  assert.equal(callbacks, 1)
+  assert.deepEqual(members.map((member) => member.opens), [0, 0, 0])
+  await set.close()
+})
+
+test("close fails a pending socket without allocating a member and rejects later opens", async () => {
+  const {set, members} = fixture()
+  const start = set.start()
+  /** @type {import('../../../src/client/virtual-socket.js').VirtualSocket | undefined} */
+  let pendingSocket
+  const callbackResult = new Promise((resolve) => {
+    pendingSocket = set.createConnection({host: "target.internal", port: 443}, (error, socket) => resolve({error, socket}))
+  })
+  assert(pendingSocket !== undefined)
+  const closed = onceEvent(pendingSocket, "close")
+  const close = set.close()
+  await assert.rejects(start, (error) => /** @type {{code?: string}} */ (error).code === "SOCKETDUCT_TRANSPORT_SET_CLOSED")
+  const result = /** @type {{error: Error, socket: import('../../../src/client/virtual-socket.js').VirtualSocket}} */ (await callbackResult)
+  await closed
+  assert.equal(/** @type {{code?: string}} */ (result.error).code, "SOCKETDUCT_TRANSPORT_SET_CLOSED")
+  await close
+  assert.deepEqual(members.map((member) => member.opens), [0, 0, 0])
+  assert.throws(() => set.createConnection({host: "target.internal", port: 443}), /closed/i)
+})
+
+test("bounds pending open count and pending application bytes", async () => {
+  const countFixture = fixture({maxPendingOpens: 1})
+  const first = countFixture.set.createConnection({host: "target.internal", port: 443})
+  assert.throws(
+    () => countFixture.set.createConnection({host: "target.internal", port: 443}),
+    (error) => error instanceof SocketductTransportSetLimitError && error.code === "SOCKETDUCT_SET_PENDING_OPEN_LIMIT"
+  )
+  const firstClosed = onceEvent(first, "close")
+  first.destroy()
+  await firstClosed
+  await countFixture.set.close()
+
+  const byteFixture = fixture({maxTotalBufferBytes: 4, maxTotalQueuedBytes: 4})
+  const socket = byteFixture.set.createConnection({host: "target.internal", port: 443})
+  const streamError = onceEvent(socket, "error")
+  const socketClosed = onceEvent(socket, "close")
+  socket.write("four")
+  const callbackError = new Promise((resolve) => {
+    assert.equal(socket.write("!", (error) => resolve(error)), false)
+  })
+  const [error] = await streamError
+  assert.equal(error, await callbackError)
+  await socketClosed
+  assert.equal(byteFixture.set.status().budgets.dataBytes, 0)
+  assert.equal(byteFixture.set.status().budgets.queuedBytes, 0)
+  await byteFixture.set.close()
+})
+
+test("pending sockets retain timeout and Writable backpressure behavior", async () => {
+  const {set, members} = fixture()
+  const started = set.start()
+  const socket = set.createConnection({
+    host: "target.internal",
+    port: 443,
+    writableHighWaterMark: 4
+  })
+  const timedOut = onceEvent(socket, "timeout")
+  socket.setTimeout(1)
+  await timedOut
+  assert.equal(socket.destroyed, false)
+  assert.equal(socket.connecting, true)
+  socket.setTimeout(0)
+
+  const drained = onceEvent(socket, "drain")
+  const written = new Promise((resolve, reject) => {
+    assert.equal(socket.write("four", (error) => error ? reject(error) : resolve(undefined)), false)
+  })
+  assert.equal(set.status().budgets.dataBytes, 4)
+  at(members, 0).ready()
+  await Promise.all([started, written, drained])
+  assert.equal(set.status().budgets.dataBytes, 0)
+  socket.destroy()
+  await set.close()
+})
+
+test("pending sockets reserve and write non-Uint8 typed-array views", async () => {
+  const payload = new Uint16Array([0x1234, 0xabcd])
+  const expected = Buffer.from(payload.buffer, payload.byteOffset, payload.byteLength)
+  const {set, members} = fixture({maxTotalBufferBytes: payload.byteLength, maxTotalQueuedBytes: payload.byteLength})
+  const started = set.start()
+  const socket = set.createConnection({host: "target.internal", port: 443, writableHighWaterMark: payload.byteLength})
+  let callbacks = 0
+  /** @type {(error?: Error | null) => void} */
+  let completeWrite = () => {}
+  const written = new Promise((resolve, reject) => {
+    completeWrite = (error) => {
+      callbacks += 1
+      if (error) reject(error)
+      else resolve(undefined)
+    }
+  })
+  const drained = onceEvent(socket, "drain")
+
+  assert.equal(socket.write(payload, completeWrite), false)
+  assert.equal(set.status().budgets.dataBytes, payload.byteLength)
+  assert.throws(
+    () => socket.write(/** @type {never} */ ({unsupported: true})),
+    (error) => /** @type {{code?: string}} */ (error).code === "ERR_INVALID_ARG_TYPE"
+  )
+  assert.equal(set.status().budgets.dataBytes, payload.byteLength)
+
+  at(members, 0).ready()
+  await Promise.all([started, written, drained])
+  assert.equal(callbacks, 1)
+  assert.deepEqual(at(at(members, 0).sockets, 0).writes, [expected])
+  assert.equal(set.status().budgets.dataBytes, 0)
+  socket.destroy()
+  await set.close()
+})
+
+test("pending socket end payload reserves and sends bytes after assignment", async () => {
+  const {set, members} = fixture({maxTotalBufferBytes: 5, maxTotalQueuedBytes: 5})
+  const started = set.start()
+  const socket = set.createConnection({host: "target.internal", port: 443})
+  let callbacks = 0
+  /** @type {string[]} */
+  const events = []
+  const finished = new Promise((resolve) => socket.once("finish", () => {
+    events.push("finish")
+    resolve(undefined)
+  }))
+  const ended = new Promise((resolve) => {
+    const returned = socket.end("final", () => {
+      callbacks += 1
+      events.push("callback")
+      resolve(undefined)
+    })
+    assert.equal(returned, socket)
+  })
+
+  assert.equal(set.status().budgets.dataBytes, 5)
+
+  at(members, 0).ready()
+  await Promise.all([started, ended, finished])
+  assert.equal(callbacks, 1)
+  assert.deepEqual(events, ["callback", "finish"])
+  assert.deepEqual(at(at(members, 0).sockets, 0).writes, [Buffer.from("final")])
+  assert.equal(set.status().budgets.dataBytes, 0)
+  await set.close()
+})
+
+test("pending socket end reserves and sends DataView bytes", async () => {
+  const buffer = new ArrayBuffer(6)
+  const payload = new DataView(buffer, 1, 4)
+  payload.setUint32(0, 0x1234abcd)
+  const expected = Buffer.from(buffer, payload.byteOffset, payload.byteLength)
+  const {set, members} = fixture({maxTotalBufferBytes: payload.byteLength, maxTotalQueuedBytes: payload.byteLength})
+  const started = set.start()
+  const socket = set.createConnection({host: "target.internal", port: 443})
+  let callbacks = 0
+  /** @type {() => void} */
+  let completeEnd = () => {}
+  const ended = new Promise((resolve) => {
+    completeEnd = () => {
+      callbacks += 1
+      resolve(undefined)
+    }
+  })
+
+  assert.equal(socket.end(payload, completeEnd), socket)
+  assert.equal(set.status().budgets.dataBytes, payload.byteLength)
+
+  at(members, 0).ready()
+  await Promise.all([started, ended])
+  assert.equal(callbacks, 1)
+  assert.deepEqual(at(at(members, 0).sockets, 0).writes, [expected])
+  assert.equal(set.status().budgets.dataBytes, 0)
+  await set.close()
+})
+
+test("recovery requires an original configured gateway and registration before start", async () => {
+  const {set, members} = fixture()
+  assert.throws(() => set.recoverConnection(/** @type {Parameters<MultiplexedSocketductTransportSet['recoverConnection']>[0]} */ (/** @type {unknown} */ ({host: "target.internal", port: 443, identifier: "stable_stream_01"}))), /gatewayId/i)
+  assert.throws(() => set.recoverConnection({gatewayId: "unknown", host: "target.internal", port: 443, identifier: "stable_stream_01"}), /unknown gatewayId/i)
+  assert.throws(() => set.createConnection(/** @type {import('../../../src/client/multiplexed-connection.js').VirtualConnectionOptions} */ (/** @type {unknown} */ ({gatewayId: "zyxel", host: "target.internal", port: 443}))), /recoverConnection/i)
+  assert.throws(() => set.createConnection({recover: true, host: "target.internal", port: 443, identifier: "stable_stream_01"}), /recoverConnection/i)
+  assert.throws(() => set.createTlsConnection({gatewayId: "zyxel", host: "target.internal", port: 443}), /recoverConnection/i)
+
+  const recovered = set.recoverConnection({
+    gatewayId: "zyxel",
+    host: "target.internal",
+    port: 443,
+    identifier: "stable_stream_01",
+    sessionName: "durable"
+  })
+  await new Promise((resolve) => setImmediate(resolve))
+  const recoveredLater = set.recoverConnection({
+    gatewayId: "zyxel",
+    host: "target.internal",
+    port: 443,
+    identifier: "stable_stream_02",
+    sessionName: "durable-later"
+  })
+  assert.equal(gatewayOf(recovered), "zyxel")
+  assert.equal(gatewayOf(recoveredLater), "zyxel")
+  assert.deepEqual(members.map((member) => member.opens), [0, 2, 0])
+  assert.equal(at(members, 1).options.connectionId === undefined, false)
+  assert.equal(at(members, 1).createConnectionCalls.length, 0)
+  assert.equal(/** @type {{recover?: unknown}} */ (at(members, 1).recoveryConnectionCalls[0]).recover, true)
+  assert.equal(at(members, 1).recoveryConnectionCalls.length, 2)
+  assert.equal(Object.hasOwn(/** @type {object} */ (at(members, 1).recoveryConnectionCalls[0]), "gatewayId"), false)
+
+  const starting = set.start()
+  assert.throws(() => set.recoverConnection({gatewayId: "zyxel", host: "target.internal", port: 443, identifier: "stable_stream_04"}), /before start/i)
+  members.forEach((member) => member.ready())
+  await starting
+  await set.close()
+})
+
+test("returns the stable selected gatewayId on ordinary and recovered sockets", async () => {
+  const {set, members} = fixture()
+  const started = set.start()
+  members.forEach((member) => member.ready())
+  await started
+  const ordinary = set.createConnection({host: "target.internal", port: 443})
+  assert.equal(gatewayOf(ordinary), "starlink")
+  const descriptor = Object.getOwnPropertyDescriptor(ordinary, "gatewayId")
+  assert.equal(descriptor?.writable, false)
+  await set.close()
+
+  const recoveryFixture = fixture()
+  const recovered = recoveryFixture.set.recoverConnection({gatewayId: "zyxel", host: "target.internal", port: 443, identifier: "stable_stream_03"})
+  assert.equal(gatewayOf(recovered), "zyxel")
+  await recoveryFixture.set.close()
+})
+
+test("shares aggregate data, spool, and queued-byte reservations across members", async () => {
+  const {set, members} = fixture({maxTotalBufferBytes: 10, maxTotalSpoolBytes: 8, maxTotalQueuedBytes: 12})
+  const budgets = members.map((member) => {
+    const budget = member.options.aggregateBudget
+    assert(budget !== undefined)
+    return budget
+  })
+  assert.equal(new Set(budgets).size, 1)
+  at(budgets, 0).reserveData(6)
+  assert.throws(() => at(budgets, 1).reserveData(3), (error) => /** @type {{code?: string}} */ (error).code === "SOCKETDUCT_SET_SPOOL_LIMIT")
+  at(budgets, 2).reserveControl(6)
+  assert.throws(() => at(budgets, 1).reserveControl(1), (error) => /** @type {{code?: string}} */ (error).code === "SOCKETDUCT_SET_QUEUE_LIMIT")
+  assert.deepEqual(set.status().budgets, {
+    dataBytes: 6,
+    spoolBytes: 6,
+    queuedBytes: 12,
+    maxDataBytes: 10,
+    maxSpoolBytes: 8,
+    maxQueuedBytes: 12
+  })
+  at(budgets, 0).releaseMemory(6)
+  at(budgets, 0).releaseSpool(6)
+  at(budgets, 2).releaseControl(6)
+  await set.close()
+})
+
+test("close during startup cannot resurrect members and concurrent close is ordered", async () => {
+  const firstError = new Error("starlink close")
+  const thirdError = new Error("o2 close")
+  const {set, members} = fixture({}, {starlink: firstError, o2gateway: thirdError})
+  const start = set.start()
+  const firstClose = set.close()
+  const secondClose = set.close()
+  assert.equal(firstClose, secondClose)
+  await assert.rejects(start, (error) => /** @type {{code?: string}} */ (error).code === "SOCKETDUCT_TRANSPORT_SET_CLOSED")
+  await assert.rejects(firstClose, (error) => {
+    assert(error instanceof AggregateError)
+    assert.deepEqual(error.errors, [firstError, thirdError])
+    return true
+  })
+  assert.deepEqual(members.map((member) => member.closeCalls), [1, 1, 1])
+  members.forEach((member) => member.ready())
+  assert.equal(set.status().state, "closed")
+  assert.throws(() => set.createConnection({host: "target.internal", port: 443}), /closed/i)
+})