socketduct 0.1.15

package/test/integration/multiplexed-failures.test.js

@@ -0,0 +1,339 @@
+/* eslint-disable jsdoc/escape-inline-tags, jsdoc/require-param-type, jsdoc/require-returns */
+import assert from "node:assert/strict"
+import {createConnection, createServer} from "node:net"
+import {Duplex} from "node:stream"
+import test from "node:test"
+
+import {StaticTokenStore} from "../../src/auth/static-token-store.js"
+import {MultiplexedSocketductConnection} from "../../src/client/multiplexed-connection.js"
+import {SocketductSessionLostError, SocketductStreamError} from "../../src/client/virtual-socket.js"
+import {startMultiplexedTcpRelay} from "../../src/proxy/multiplexed-relay.js"
+
+/** @type {WeakMap<import('node:net').Server, Set<import('node:net').Socket>>} */
+const socketsByServer = new WeakMap()
+
+/** @param {import('node:net').Server} server - Server. @returns {Promise<number>} */
+async function listen(server) {
+  const sockets = new Set()
+  socketsByServer.set(server, sockets)
+  server.on("connection", (socket) => {
+    sockets.add(socket)
+    socket.once("close", () => sockets.delete(socket))
+  })
+  await new Promise((resolve, reject) => {
+    server.once("error", reject)
+    server.listen(0, "127.0.0.1", () => { server.off("error", reject); resolve(undefined) })
+  })
+  const address = server.address()
+  assert(address !== null && typeof address !== "string")
+  return address.port
+}
+
+/** @param {import('node:net').Server} server - Server. @returns {Promise<void>} */
+async function closeServer(server) {
+  for (const socket of socketsByServer.get(server) ?? []) socket.destroy()
+  if (server.listening) await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve(undefined)))
+}
+
+/**
+ * @param {import('node:events').EventEmitter} emitter - Emitter. @param {string} event - Event. @returns {Promise<unknown[]>}
+ * @param event
+ */
+async function onceEvent(emitter, event) {
+  return await new Promise((resolve, reject) => {
+    emitter.once("error", reject)
+    /** @param {...unknown} values - Event values. */
+    const onEvent = (...values) => { emitter.off("error", reject); resolve(values) }
+    emitter.once(event, onEvent)
+  })
+}
+
+/** @param {import('node:stream').Duplex} socket - Socket. @returns {Promise<Error & {code?: string}>} */
+async function nextError(socket) {
+  return await new Promise((resolve) => socket.once("error", (error) => resolve(/** @type {Error & {code?: string}} */ (error))))
+}
+
+/** @param {import('node:stream').Duplex} socket - Socket. @returns {Promise<string>} */
+async function readText(socket) {
+  return Buffer.from(/** @type {Uint8Array} */ ((await onceEvent(socket, "data"))[0])).toString()
+}
+
+/**
+ * @param {() => boolean} predicate - Condition. @param {number} [timeoutMs] - Deadline. @returns {Promise<void>}
+ * @param timeoutMs
+ */
+async function waitUntil(predicate, timeoutMs = 2_000) {
+  const deadline = Date.now() + timeoutMs
+  while (!predicate()) {
+    if (Date.now() >= deadline) throw new Error("timed out waiting for condition")
+    await new Promise((resolve) => setTimeout(resolve, 5))
+  }
+}
+
+/**
+ *
+ */
+function tokens() {
+  return new StaticTokenStore([{username: "edge", token: "valid-token"}])
+}
+
+test("authentication and exact-target authorization fail closed", async (t) => {
+  const relay = await startMultiplexedTcpRelay({tokens: tokens(), authorizeTarget: () => false})
+  t.after(() => relay.close())
+  const unauthorizedConnection = new MultiplexedSocketductConnection({
+    relay: {host: "127.0.0.1", port: relay.port, token: "valid-token"}
+  })
+  t.after(() => unauthorizedConnection.close())
+  const denied = unauthorizedConnection.createConnection({host: "127.0.0.1", port: 9})
+  const deniedError = await nextError(denied)
+  assert(deniedError instanceof SocketductStreamError)
+  assert.equal(deniedError.code, "TARGET_UNAUTHORIZED")
+  assert.equal(relay.status().activeTargets, 0)
+
+  const invalidConnection = new MultiplexedSocketductConnection({
+    relay: {host: "127.0.0.1", port: relay.port, token: "wrong-token"},
+    maxReconnectAttempts: 2
+  })
+  t.after(() => invalidConnection.close())
+  const invalid = invalidConnection.createConnection({host: "127.0.0.1", port: 9})
+  const authError = await nextError(invalid)
+  assert(authError instanceof SocketductSessionLostError)
+  assert.equal(authError.code, "AUTH_REQUIRED")
+  await new Promise((resolve) => setTimeout(resolve, 30))
+  assert.equal(invalidConnection.status().reconnectAttempts, 0)
+})
+
+test("target open failure and stream reset are isolated from siblings", async (t) => {
+  const echo = createServer((socket) => socket.on("data", (chunk) => socket.write(chunk)))
+  const echoPort = await listen(echo)
+  t.after(() => closeServer(echo))
+  const unused = createServer()
+  const failedPort = await listen(unused)
+  await closeServer(unused)
+  const relay = await startMultiplexedTcpRelay({
+    tokens: tokens(),
+    authorizeTarget: ({target}) => target.host === "127.0.0.1" && (target.port === echoPort || target.port === failedPort)
+  })
+  t.after(() => relay.close())
+  const connection = new MultiplexedSocketductConnection({relay: {host: "127.0.0.1", port: relay.port, token: "valid-token"}})
+  t.after(() => connection.close())
+  const failed = connection.createConnection({host: "127.0.0.1", port: failedPort})
+  const sibling = connection.createConnection({host: "127.0.0.1", port: echoPort})
+  const failure = await nextError(failed)
+  assert.equal(failure.code, "TARGET_OPEN_FAILED")
+  await onceEvent(sibling, "connect")
+  const response = readText(sibling)
+  sibling.write("still-open")
+  assert.equal(await response, "still-open")
+  assert.equal(relay.status().physicalConnections, 1)
+  sibling.destroy()
+})
+
+test("resume loss resets only the affected stream while its sibling resumes", async (t) => {
+  /** @type {import('node:net').Socket[]} */
+  const accepted = []
+  const target = createServer((socket) => {
+    accepted.push(socket)
+    socket.on("data", (chunk) => socket.write(chunk))
+  })
+  const targetPort = await listen(target)
+  t.after(() => closeServer(target))
+  const relay = await startMultiplexedTcpRelay({
+    tokens: tokens(),
+    authorizeTarget: ({target: requested}) => requested.port === targetPort,
+    resumeWindowMs: 2_000
+  })
+  t.after(() => relay.close())
+  const connection = new MultiplexedSocketductConnection({
+    relay: {host: "127.0.0.1", port: relay.port, token: "valid-token"},
+    reconnectDelayMs: 40
+  })
+  t.after(() => connection.close())
+  const lost = connection.createConnection({host: "127.0.0.1", port: targetPort, sessionName: "lost"})
+  const survivor = connection.createConnection({host: "127.0.0.1", port: targetPort, sessionName: "survivor"})
+  await Promise.all([onceEvent(lost, "connect"), onceEvent(survivor, "connect")])
+  assert.equal(accepted.length, 2)
+  const lostError = nextError(lost)
+  const survivorReady = onceEvent(survivor, "ready")
+  await relay.closeActiveTransport()
+  const targetToLose = accepted[0]
+  assert(targetToLose !== undefined)
+  targetToLose.destroy()
+  const error = await lostError
+  assert(error instanceof SocketductSessionLostError)
+  assert.equal(error.code, "RESUME_SESSION_LOST")
+  await survivorReady
+  const response = readText(survivor)
+  survivor.write("resumed")
+  assert.equal(await response, "resumed")
+  assert.equal(connection.status().reconnectAttempts, 1)
+  survivor.destroy()
+})
+
+test("a drop between OPEN and OPENED resumes the same stream ID safely", async (t) => {
+  /** @type {Duplex | undefined} */
+  let delayedTarget
+  const relay = await startMultiplexedTcpRelay({
+    tokens: tokens(),
+    authorizeTarget: () => true,
+    createTargetConnection: () => {
+      delayedTarget = new Duplex({
+        read() {},
+        write(_chunk, _encoding, callback) { callback() }
+      })
+      return /** @type {import('node:net').Socket} */ (/** @type {unknown} */ (delayedTarget))
+    }
+  })
+  t.after(() => relay.close())
+  const connection = new MultiplexedSocketductConnection({
+    relay: {host: "127.0.0.1", port: relay.port, token: "valid-token"},
+    reconnectDelayMs: 30
+  })
+  t.after(() => connection.close())
+  const socket = connection.createConnection({host: "target.internal", port: 1234})
+  const connected = onceEvent(socket, "connect")
+  await waitUntil(() => delayedTarget !== undefined)
+  await relay.closeActiveTransport()
+  delayedTarget?.emit("connect")
+  await connected
+  assert.equal(socket.streamId, 1)
+  assert.equal(connection.status().reconnectAttempts, 1)
+  assert.equal(relay.status().resumeSuccesses, 1)
+  socket.destroy()
+})
+
+test("bounded per-stream and aggregate queues fail only the offending stream", async (t) => {
+  const target = createServer((socket) => socket.pause())
+  const targetPort = await listen(target)
+  t.after(() => closeServer(target))
+  const relay = await startMultiplexedTcpRelay({tokens: tokens(), authorizeTarget: ({target: requested}) => requested.port === targetPort})
+  t.after(() => relay.close())
+  const connection = new MultiplexedSocketductConnection({
+    relay: {host: "127.0.0.1", port: relay.port, token: "valid-token"},
+    maxStreamBufferBytes: 32,
+    maxConnectionBufferBytes: 48,
+    maxFrameBytes: 16,
+    initialWindowBytes: 32
+  })
+  t.after(() => connection.close())
+  const offender = connection.createConnection({host: "127.0.0.1", port: targetPort})
+  const sibling = connection.createConnection({host: "127.0.0.1", port: targetPort})
+  await Promise.all([onceEvent(offender, "connect"), onceEvent(sibling, "connect")])
+  const errorPromise = nextError(offender)
+  offender.write(Buffer.alloc(33))
+  const error = await errorPromise
+  assert.equal(error.code, "SOCKETDUCT_BUFFER_LIMIT")
+  assert.equal(sibling.destroyed, false)
+  assert(connection.status().dataQueuedBytes <= 48)
+  assert(connection.status().controlQueuedBytes <= 1024 * 1024)
+  sibling.destroy()
+})
+
+test("round-robin data scheduling lets a small control flow pass a large upload", async (t) => {
+  let largeBytesObserved = 0
+  let largeCompleted = false
+  let smallObservedAfterLargeCompleted
+  const largeTarget = createServer((socket) => socket.on("data", (chunk) => {
+    largeBytesObserved += chunk.length
+    if (largeBytesObserved === 64 * 1024) largeCompleted = true
+  }))
+  const largePort = await listen(largeTarget)
+  t.after(() => closeServer(largeTarget))
+  const smallTarget = createServer((socket) => socket.on("data", (chunk) => {
+    smallObservedAfterLargeCompleted = largeCompleted
+    socket.write(chunk)
+  }))
+  const smallPort = await listen(smallTarget)
+  t.after(() => closeServer(smallTarget))
+  const relay = await startMultiplexedTcpRelay({
+    tokens: tokens(),
+    authorizeTarget: ({target}) => target.port === largePort || target.port === smallPort,
+    maxFrameBytes: 1024
+  })
+  t.after(() => relay.close())
+  const connection = new MultiplexedSocketductConnection({
+    relay: {host: "127.0.0.1", port: relay.port, token: "valid-token"},
+    maxFrameBytes: 1024
+  })
+  t.after(() => connection.close())
+  const large = connection.createConnection({host: "127.0.0.1", port: largePort})
+  const small = connection.createConnection({host: "127.0.0.1", port: smallPort})
+  await Promise.all([onceEvent(large, "connect"), onceEvent(small, "connect")])
+  const smallResponse = readText(small)
+  large.write(Buffer.alloc(64 * 1024))
+  small.write("control")
+  assert.equal(await smallResponse, "control")
+  assert.equal(smallObservedAfterLargeCompleted, false)
+  assert(largeBytesObserved < 64 * 1024)
+  large.destroy()
+  small.destroy()
+})
+
+test("per-stream receive windows bound a paused consumer and resume independently", async (t) => {
+  const payload = Buffer.alloc(4 * 1024, 0x61)
+  const target = createServer((socket) => socket.write(payload))
+  const targetPort = await listen(target)
+  t.after(() => closeServer(target))
+  const relay = await startMultiplexedTcpRelay({
+    tokens: tokens(),
+    authorizeTarget: ({target: requested}) => requested.port === targetPort,
+    initialWindowBytes: 1024,
+    maxFrameBytes: 512,
+    maxStreamBufferBytes: 8 * 1024
+  })
+  t.after(() => relay.close())
+  const connection = new MultiplexedSocketductConnection({
+    relay: {host: "127.0.0.1", port: relay.port, token: "valid-token"},
+    initialWindowBytes: 1024,
+    maxFrameBytes: 512
+  })
+  t.after(() => connection.close())
+  const paused = connection.createConnection({
+    host: "127.0.0.1",
+    port: targetPort,
+    readableHighWaterMark: 256
+  })
+  await onceEvent(paused, "connect")
+  await waitUntil(() => relay.status().queuedBytes === payload.length)
+  assert.equal(relay.status().resets, 0)
+  assert(relay.status().queuedBytes <= 8 * 1024)
+
+  const received = await new Promise((resolve, reject) => {
+    /** @type {Buffer[]} */
+    const chunks = []
+    let bytes = 0
+    paused.on("data", (chunk) => {
+      chunks.push(chunk)
+      bytes += chunk.length
+      if (bytes === payload.length) resolve(Buffer.concat(chunks))
+    })
+    paused.once("error", reject)
+    paused.resume()
+  })
+  assert.deepEqual(received, payload)
+  await waitUntil(() => relay.status().queuedBytes === 0)
+  paused.destroy()
+})
+
+test("relay accepts local target sockets without creating extra WAN transports", async (t) => {
+  const target = createServer((socket) => socket.end())
+  const port = await listen(target)
+  t.after(() => closeServer(target))
+  let localConnects = 0
+  const relay = await startMultiplexedTcpRelay({
+    tokens: tokens(),
+    authorizeTarget: ({target: requested}) => requested.port === port,
+    createTargetConnection: (requested) => {
+      localConnects += 1
+      return createConnection(requested)
+    }
+  })
+  t.after(() => relay.close())
+  const connection = new MultiplexedSocketductConnection({relay: {host: "127.0.0.1", port: relay.port, token: "valid-token"}})
+  t.after(() => connection.close())
+  const streams = Array.from({length: 4}, () => connection.createConnection({host: "127.0.0.1", port}))
+  await Promise.all(streams.map((socket) => onceEvent(socket, "connect")))
+  assert.equal(localConnects, 4)
+  assert.equal(relay.status().physicalAccepts, 1)
+  streams.forEach((socket) => socket.destroy())
+})