socketduct 0.1.15

package/test/e2e/gateway-process.test.js

@@ -0,0 +1,262 @@
+import assert from "node:assert/strict"
+import {spawn} from "node:child_process"
+import {mkdtemp, rm} from "node:fs/promises"
+import {createConnection, createServer} from "node:net"
+import {tmpdir} from "node:os"
+import {join} from "node:path"
+import test from "node:test"
+
+import {startTcpEdge} from "../../src/edge/tcp-edge.js"
+import {encodeFrame, FrameDecoder} from "../../src/protocol/frame.js"
+import {SessionSpool} from "../../src/spool/spool-file.js"
+
+const PROCESS_TIMEOUT_MS = 5_000
+
+/**
+ * @param {import('node:net').Server} server - Server to bind.
+ * @returns {Promise<number>} Bound port.
+ */
+async function listen(server) {
+  await new Promise((resolve, reject) => {
+    server.once("error", reject)
+    server.listen(0, "127.0.0.1", () => {
+      server.off("error", reject)
+      resolve(undefined)
+    })
+  })
+  const address = server.address()
+  if (address === null || typeof address === "string") {
+    throw new Error("Server did not bind to a TCP port")
+  }
+  return address.port
+}
+
+/** @returns {Promise<number>} Available TCP port. */
+async function reservePort() {
+  const server = createServer()
+  const port = await listen(server)
+  await closeServer(server)
+  return port
+}
+
+/**
+ * @param {import('node:net').Server} server - Server to close.
+ * @returns {Promise<void>} Close completion.
+ */
+async function closeServer(server) {
+  await new Promise((resolve, reject) => {
+    server.close((error) => {
+      if (error) {
+        reject(error)
+        return
+      }
+      resolve(undefined)
+    })
+  })
+}
+
+/**
+ * @param {import('node:net').Socket} socket - Socket to wait on.
+ * @returns {Promise<void>} Connect completion.
+ */
+async function waitConnect(socket) {
+  await new Promise((resolve, reject) => {
+    socket.once("error", reject)
+    socket.once("connect", () => {
+      socket.off("error", reject)
+      resolve(undefined)
+    })
+  })
+}
+
+/**
+ * @param {import('node:net').Socket} socket - Socket to read.
+ * @returns {Promise<Buffer>} Next chunk.
+ */
+async function readOnce(socket) {
+  return await withTimeout(new Promise((resolve) => {
+    socket.once("data", (chunk) => resolve(chunk))
+  }), PROCESS_TIMEOUT_MS, "Timed out waiting for socket data")
+}
+
+/**
+ * @param {Promise<T>} promise - Promise to wrap.
+ * @param {number} timeoutMs - Timeout in milliseconds.
+ * @param {string} message - Timeout message.
+ * @template T
+ * @returns {Promise<T>} Promise result.
+ */
+async function withTimeout(promise, timeoutMs, message) {
+  /** @type {ReturnType<typeof setTimeout> | undefined} */
+  let timer
+  const timeout = new Promise((_, reject) => {
+    timer = setTimeout(() => reject(new Error(message)), timeoutMs)
+  })
+  try {
+    return await Promise.race([promise, timeout])
+  } finally {
+    if (timer !== undefined) {
+      clearTimeout(timer)
+    }
+  }
+}
+
+/**
+ * @param {Record<string, string>} env - Relay process environment.
+ * @returns {Promise<{process: import('node:child_process').ChildProcessByStdio<null, import('node:stream').Readable, import('node:stream').Readable>, output: () => string}>} Started relay process.
+ */
+async function startRelayProcess(env) {
+  const child = spawn(process.execPath, ["src/index.js"], {
+    cwd: process.cwd(),
+    env: {...process.env, ...env},
+    stdio: ["ignore", "pipe", "pipe"]
+  })
+  let output = ""
+  child.stdout.on("data", (chunk) => { output += chunk.toString("utf8") })
+  child.stderr.on("data", (chunk) => { output += chunk.toString("utf8") })
+
+  try {
+    await withTimeout(new Promise((resolve, reject) => {
+      const cleanup = () => {
+        child.off("exit", onExit)
+        child.stdout.off("data", onStdout)
+      }
+      /** @type {(code: number | null) => void} */
+      const onExit = (code) => {
+        cleanup()
+        reject(new Error(`Relay exited before listening with ${code}: ${output}`))
+      }
+      const onStdout = () => {
+        if (output.includes('"event":"relay.started"')) {
+          cleanup()
+          resolve(undefined)
+        }
+      }
+
+      child.once("exit", onExit)
+      child.stdout.on("data", onStdout)
+      onStdout()
+    }), PROCESS_TIMEOUT_MS, `Timed out waiting for relay startup: ${output}`)
+  } catch (error) {
+    await stopProcess(child)
+    throw error
+  }
+
+  return {process: child, output: () => output}
+}
+
+/**
+ * @param {import('node:child_process').ChildProcess} child - Child process.
+ * @returns {Promise<void>} Stop completion.
+ */
+async function stopProcess(child) {
+  if (child.exitCode !== null || child.killed) {
+    return
+  }
+  child.kill("SIGTERM")
+  await withTimeout(new Promise((resolve) => {
+    child.once("exit", () => resolve(undefined))
+  }), PROCESS_TIMEOUT_MS, "Timed out waiting for relay shutdown")
+}
+
+test("e2e process gateway forwards edge traffic to an allowlisted target", async () => {
+  const tempDir = await mkdtemp(join(tmpdir(), "socketduct-e2e-spool-"))
+  const targetServer = createServer((socket) => {
+    socket.on("data", (chunk) => socket.write(Buffer.concat([Buffer.from("echo:"), chunk])))
+  })
+  const targetPort = await listen(targetServer)
+  const relayPort = await reservePort()
+  const relay = await startRelayProcess({
+    SOCKETDUCT_MODE: "relay",
+    SOCKETDUCT_RELAY_HOST: "127.0.0.1",
+    SOCKETDUCT_RELAY_PORT: String(relayPort),
+    SOCKETDUCT_AUTH_TOKENS: "tensorbuzz-edge:e2e-token-not-placeholder",
+    SOCKETDUCT_TARGET_ALLOWLIST: `127.0.0.1:${targetPort}`
+  })
+
+  const edge = await startTcpEdge({
+    relayHost: "127.0.0.1",
+    relayPort,
+    token: "e2e-token-not-placeholder",
+    sessionName: "e2e process gateway",
+    target: {host: "127.0.0.1", port: targetPort},
+    spool: new SessionSpool({rootDir: tempDir}),
+    reconnectDelayMs: 10
+  })
+
+  test.after(async () => {
+    await edge.close()
+    await stopProcess(relay.process)
+    await closeServer(targetServer)
+    await rm(tempDir, {recursive: true, force: true})
+  })
+
+  const appSocket = createConnection({host: "127.0.0.1", port: edge.port})
+  await waitConnect(appSocket)
+  appSocket.write("hello-tensorbuzz")
+  assert.equal((await readOnce(appSocket)).toString("utf8"), "echo:hello-tensorbuzz")
+  appSocket.destroy()
+})
+
+test("e2e process gateway denies non-allowlisted target requests", async () => {
+  const allowedTarget = createServer((socket) => {
+    socket.on("data", (chunk) => socket.write(chunk))
+  })
+  const allowedPort = await listen(allowedTarget)
+  const blockedPort = await reservePort()
+  const relayPort = await reservePort()
+  const relay = await startRelayProcess({
+    SOCKETDUCT_MODE: "relay",
+    SOCKETDUCT_RELAY_HOST: "127.0.0.1",
+    SOCKETDUCT_RELAY_PORT: String(relayPort),
+    SOCKETDUCT_AUTH_TOKENS: "tensorbuzz-edge:e2e-token-not-placeholder",
+    SOCKETDUCT_TARGET_ALLOWLIST: `127.0.0.1:${allowedPort}`
+  })
+
+  test.after(async () => {
+    await stopProcess(relay.process)
+    await closeServer(allowedTarget)
+  })
+
+  const tunnel = createConnection({host: "127.0.0.1", port: relayPort})
+  const decoder = new FrameDecoder()
+  await waitConnect(tunnel)
+  tunnel.write(encodeFrame({
+    type: "session.create",
+    token: "e2e-token-not-placeholder",
+    name: "blocked process gateway target",
+    target: {host: "127.0.0.1", port: blockedPort},
+    framing: [2]
+  }))
+
+  const [frame] = decoder.push(await readOnce(tunnel))
+  assert.equal(frame?.header.type, "error")
+  assert.equal(frame?.header.code, "TARGET_UNAUTHORIZED")
+  tunnel.destroy()
+})
+
+test("e2e process gateway rejects placeholder tokens before listening", async () => {
+  const relayPort = await reservePort()
+  const child = spawn(process.execPath, ["src/index.js"], {
+    cwd: process.cwd(),
+    env: {
+      ...process.env,
+      SOCKETDUCT_MODE: "relay",
+      SOCKETDUCT_RELAY_HOST: "127.0.0.1",
+      SOCKETDUCT_RELAY_PORT: String(relayPort),
+      SOCKETDUCT_AUTH_TOKENS: "tensorbuzz-edge:replace-with-generated-token",
+      SOCKETDUCT_TARGET_ALLOWLIST: "127.0.0.1:443"
+    },
+    stdio: ["ignore", "pipe", "pipe"]
+  })
+  let output = ""
+  child.stdout.on("data", (chunk) => { output += chunk.toString("utf8") })
+  child.stderr.on("data", (chunk) => { output += chunk.toString("utf8") })
+
+  const exitCode = await withTimeout(new Promise((resolve) => {
+    child.once("exit", (code) => resolve(code))
+  }), PROCESS_TIMEOUT_MS, "Timed out waiting for placeholder-token process exit")
+
+  assert.notEqual(exitCode, 0)
+  assert.match(output, /placeholder tokens/)
+})

package/test/e2e/reverse-gateway-process.test.js

@@ -0,0 +1,285 @@
+/* eslint-disable jsdoc/escape-inline-tags, jsdoc/require-param, jsdoc/require-returns */
+import assert from "node:assert/strict"
+import {spawn} from "node:child_process"
+import {mkdtemp, readFile, readdir, rm} from "node:fs/promises"
+import {createServer} from "node:net"
+import {tmpdir} from "node:os"
+import {join} from "node:path"
+import test from "node:test"
+
+import {ReverseGatewayConnection} from "../../src/reverse/client.js"
+import {generateTestCertificates} from "../helpers/tls-certificates.js"
+
+const PROCESS_TIMEOUT_MS = 8_000
+const PROCESS_LISTEN_ATTEMPTS = 3
+
+/** A child exit observed before its expected startup event. */
+class ProcessStartupExitError extends Error {
+  /**
+   * @param {number | null} code - Exit code.
+   * @param {string} event - Expected startup event.
+   * @param {string} output - Complete child output.
+   * @param {import('node:child_process').ChildProcess} child - Exited child.
+   */
+  constructor(code, event, output, child) {
+    super(`Socketduct process exited ${String(code)} before ${event}: ${output}`)
+    this.name = "ProcessStartupExitError"
+    this.child = child
+    this.reason = isListenAddressInUse(output) ? "listen-address-in-use" : "process-exit"
+  }
+}
+
+/** @param {string} output - Complete child output. @returns {boolean} Whether Node reported listen EADDRINUSE. */
+function isListenAddressInUse(output) {
+  return /^Error: listen EADDRINUSE:/mu.test(output) &&
+    /^\s*code: ['"]EADDRINUSE['"],?\s*$/mu.test(output) &&
+    /^\s*syscall: ['"]listen['"],?\s*$/mu.test(output)
+}
+
+/** @param {import('node:net').Server} server - Server. */
+async function listen(server) {
+  await new Promise((resolve, reject) => {
+    server.once("error", reject)
+    server.listen(0, "127.0.0.1", () => { server.off("error", reject); resolve(undefined) })
+  })
+  const address = server.address()
+  assert(address !== null && typeof address !== "string")
+  return address.port
+}
+
+/** Reserves a currently unused loopback port. */
+async function reservePort() {
+  const server = createServer()
+  const port = await listen(server)
+  await new Promise((resolve) => server.close(() => resolve(undefined)))
+  return port
+}
+
+/** @param {Record<string, string>} environment - Runtime environment. @param {string} event - Expected startup event. */
+async function startProcess(environment, event) {
+  const child = spawn(process.execPath, ["src/index.js"], {
+    cwd: process.cwd(),
+    env: {...process.env, ...environment},
+    stdio: ["ignore", "pipe", "pipe"]
+  })
+  let output = ""
+  child.stdout.on("data", (chunk) => { output += chunk.toString() })
+  child.stderr.on("data", (chunk) => { output += chunk.toString() })
+  try {
+    await withTimeout(new Promise((resolve, reject) => {
+      const onClose = (/** @type {number | null} */ code) => reject(new ProcessStartupExitError(code, event, output, child))
+      const onData = () => {
+        if (!output.includes(`"event":"${event}"`)) return
+        child.off("close", onClose)
+        child.stdout.off("data", onData)
+        resolve(undefined)
+      }
+      child.once("close", onClose)
+      child.stdout.on("data", onData)
+      onData()
+    }), `process event ${event}`)
+  } catch (error) {
+    await stopProcess(child)
+    throw error
+  }
+  return child
+}
+
+/**
+ * Starts a listener process on a reserved port, retrying only a classified pre-start bind collision.
+ * @param {Record<string, string>} environment - Runtime environment without its listener port.
+ * @param {string} event - Expected startup event.
+ * @param {{reservePort?: () => Promise<number>, startProcess?: typeof startProcess}} [options] - Test dependencies.
+ * @returns {Promise<{child: import('node:child_process').ChildProcess, port: number, attempts: import('node:child_process').ChildProcess[]}>} Started process details.
+ */
+async function startProcessOnReservedPort(environment, event, options = {}) {
+  const reserve = options.reservePort ?? reservePort
+  const start = options.startProcess ?? startProcess
+  /** @type {import('node:child_process').ChildProcess[]} */
+  const attempts = []
+  for (let attempt = 1; attempt <= PROCESS_LISTEN_ATTEMPTS; attempt += 1) {
+    const port = await reserve()
+    try {
+      const child = await start({...environment, SOCKETDUCT_RELAY_PORT: String(port)}, event)
+      attempts.push(child)
+      return {child, port, attempts}
+    } catch (error) {
+      if (error instanceof ProcessStartupExitError) attempts.push(error.child)
+      if (!(error instanceof ProcessStartupExitError && error.reason === "listen-address-in-use") || attempt === PROCESS_LISTEN_ATTEMPTS) throw error
+    }
+  }
+  throw new Error("Listener startup attempts exhausted")
+}
+
+/** @param {import('node:child_process').ChildProcess} child - Child. */
+async function stopProcess(child) {
+  if (child.exitCode !== null || child.killed) return
+  child.kill("SIGTERM")
+  await withTimeout(new Promise((resolve) => child.once("exit", () => resolve(undefined))), "process shutdown")
+}
+
+/** @param {Promise<unknown>} promise - Work. @param {string} label - Timeout. */
+async function withTimeout(promise, label) {
+  let timer
+  try {
+    return await Promise.race([promise, new Promise((_, reject) => { timer = setTimeout(() => reject(new Error(`Timed out: ${label}`)), PROCESS_TIMEOUT_MS) })])
+  } finally {
+    if (timer !== undefined) clearTimeout(timer)
+  }
+}
+
+/** @param {import('node:events').EventEmitter} emitter - Source. @param {string} event - Event. */
+async function onceEvent(emitter, event) {
+  await new Promise((resolve, reject) => {
+    emitter.once("error", reject)
+    emitter.once(event, () => { emitter.off("error", reject); resolve(undefined) })
+  })
+}
+
+test("reserved reverse-hub startup retries a pre-start listen EADDRINUSE without leaking attempts", async (t) => {
+  const root = await mkdtemp(join(tmpdir(), "socketduct-reverse-port-retry-"))
+  t.after(() => rm(root, {recursive: true, force: true}))
+  const certificates = generateTestCertificates(root)
+  const occupiedListener = createServer()
+  const occupiedPort = await listen(occupiedListener)
+  t.after(() => new Promise((resolve) => occupiedListener.close(() => resolve(undefined))))
+  let reservations = 0
+  const {child, port, attempts} = await startProcessOnReservedPort({
+    SOCKETDUCT_MODE: "reverse-hub",
+    SOCKETDUCT_RELAY_HOST: "127.0.0.1",
+    SOCKETDUCT_TLS_CERT_FILE: certificates.serverCertFile,
+    SOCKETDUCT_TLS_KEY_FILE: certificates.serverKeyFile,
+    SOCKETDUCT_AUTH_TOKENS: "home-o2:process-gateway-token,tensorbuzz:process-client-token",
+    SOCKETDUCT_REVERSE_GATEWAY_BINDINGS: "o2:home-o2",
+    SOCKETDUCT_REVERSE_ROUTES: "tensorbuzz:o2:127.0.0.1:1",
+    SOCKETDUCT_STATUS_INTERVAL_MS: "0"
+  }, "reverse-hub.started", {
+    reservePort: async () => {
+      reservations += 1
+      return reservations === 1 ? occupiedPort : reservePort()
+    }
+  })
+  t.after(() => stopProcess(child))
+
+  assert.equal(reservations, 2)
+  assert.notEqual(port, occupiedPort)
+  assert.equal(attempts.length, 2)
+  const [failedAttempt, successfulAttempt] = attempts
+  assert(failedAttempt !== undefined)
+  assert(successfulAttempt !== undefined)
+  assert(failedAttempt.exitCode !== null || failedAttempt.signalCode !== null)
+  assert.equal(successfulAttempt, child)
+  assert.equal(child.exitCode, null)
+  assert.equal(child.signalCode, null)
+  assert.equal(occupiedListener.listening, true)
+
+  await stopProcess(child)
+  assert(child.exitCode !== null || child.signalCode !== null)
+  const releasedPortProbe = createServer()
+  t.after(() => {
+    if (!releasedPortProbe.listening) return undefined
+    return new Promise((resolve) => releasedPortProbe.close(() => resolve(undefined)))
+  })
+  await new Promise((resolve, reject) => {
+    releasedPortProbe.once("error", reject)
+    releasedPortProbe.listen(port, "127.0.0.1", () => {
+      releasedPortProbe.off("error", reject)
+      resolve(undefined)
+    })
+  })
+})
+
+test("reserved process startup does not retry an unclassified EADDRINUSE failure", async () => {
+  const failure = new Error("untrusted EADDRINUSE text is not a listener-bind classification")
+  let reservations = 0
+  let starts = 0
+  await assert.rejects(startProcessOnReservedPort({}, "reverse-hub.started", {
+    reservePort: async () => {
+      reservations += 1
+      return 31_337
+    },
+    startProcess: async () => {
+      starts += 1
+      throw failure
+    }
+  }), (error) => error === failure)
+  assert.equal(reservations, 1)
+  assert.equal(starts, 1)
+})
+
+test("spawned reverse hub and outbound gateway expose an allowlisted Docker stream over verified TLS", async (t) => {
+  const root = await mkdtemp(join(tmpdir(), "socketduct-reverse-process-"))
+  t.after(() => rm(root, {recursive: true, force: true}))
+  const certificates = generateTestCertificates(root)
+  const logDirectory = join(root, "logs")
+  const targetSockets = new Set()
+  const target = createServer((socket) => {
+    targetSockets.add(socket)
+    socket.once("close", () => targetSockets.delete(socket))
+    socket.on("data", (chunk) => socket.write(Buffer.concat([Buffer.from("docker:"), chunk])))
+  })
+  const targetPort = await listen(target)
+  t.after(() => {
+    for (const socket of targetSockets) socket.destroy()
+    return new Promise((resolve) => target.close(() => resolve(undefined)))
+  })
+  const {child: hub, port: hubPort} = await startProcessOnReservedPort({
+    SOCKETDUCT_MODE: "reverse-hub",
+    SOCKETDUCT_RELAY_HOST: "127.0.0.1",
+    SOCKETDUCT_TLS_CERT_FILE: certificates.serverCertFile,
+    SOCKETDUCT_TLS_KEY_FILE: certificates.serverKeyFile,
+    SOCKETDUCT_AUTH_TOKENS: "home-o2:process-gateway-token,tensorbuzz:process-client-token",
+    SOCKETDUCT_REVERSE_GATEWAY_BINDINGS: "o2:home-o2",
+    SOCKETDUCT_REVERSE_ROUTES: `tensorbuzz:o2:127.0.0.1:${targetPort}`,
+    SOCKETDUCT_STATUS_INTERVAL_MS: "0",
+    SOCKETDUCT_LOG_DIRECTORY: logDirectory,
+    SOCKETDUCT_LOG_MAX_FILE_BYTES: "4096",
+    SOCKETDUCT_LOG_MAX_FILES: "2"
+  }, "reverse-hub.started")
+  t.after(() => stopProcess(hub))
+  const gateway = await startProcess({
+    SOCKETDUCT_MODE: "reverse-gateway",
+    SOCKETDUCT_REVERSE_GATEWAY_ID: "o2",
+    SOCKETDUCT_REVERSE_CONNECTION_ID: "process_home_o2_gateway",
+    SOCKETDUCT_REVERSE_HUB_HOST: "127.0.0.1",
+    SOCKETDUCT_REVERSE_HUB_PORT: String(hubPort),
+    SOCKETDUCT_REVERSE_TOKEN: "process-gateway-token",
+    SOCKETDUCT_REVERSE_HUB_CA_FILE: certificates.caCertFile,
+    SOCKETDUCT_REVERSE_HUB_SERVERNAME: "localhost",
+    SOCKETDUCT_TARGET_ALLOWLIST: `127.0.0.1:${targetPort}`,
+    SOCKETDUCT_STATUS_INTERVAL_MS: "0",
+    SOCKETDUCT_LOG_DIRECTORY: logDirectory,
+    SOCKETDUCT_LOG_MAX_FILE_BYTES: "4096",
+    SOCKETDUCT_LOG_MAX_FILES: "2"
+  }, "reverse-gateway.started")
+  t.after(() => stopProcess(gateway))
+
+  const client = new ReverseGatewayConnection({
+    gatewayId: "o2",
+    connectionId: "process_tensorbuzz_client",
+    hub: {host: "127.0.0.1", port: hubPort, token: "process-client-token"},
+    tls: {caFile: certificates.caCertFile, servername: "localhost"}
+  })
+  t.after(() => client.close())
+  await withTimeout(client.connect(), "process client connect")
+  const socket = client.createConnection({host: "127.0.0.1", port: targetPort, sessionName: "docker-version-process"})
+  await withTimeout(onceEvent(socket, "connect"), "process stream open")
+  const response = new Promise((resolve, reject) => { socket.once("error", reject); socket.once("data", resolve) })
+  socket.write("GET /version")
+  assert.equal(Buffer.from(/** @type {Uint8Array} */ (await withTimeout(response, "process response"))).toString(), "docker:GET /version")
+  assert.equal(client.status().compression, "zstd")
+  socket.destroy()
+
+  assert(hub.pid !== undefined)
+  assert(gateway.pid !== undefined)
+  const files = (await readdir(logDirectory)).sort()
+  assert(files.includes(`reverse-hub-${hub.pid}.jsonl`))
+  assert(files.includes(`reverse-gateway-${gateway.pid}.jsonl`))
+  const hubEntries = (await readFile(join(logDirectory, `reverse-hub-${hub.pid}.jsonl`), "utf8")).trim().split("\n").map((line) => JSON.parse(line))
+  const gatewayContents = await readFile(join(logDirectory, `reverse-gateway-${gateway.pid}.jsonl`), "utf8")
+  const gatewayEntries = gatewayContents.trim().split("\n").map((line) => JSON.parse(line))
+  assert(hubEntries.some(({event}) => event === "process.started"))
+  assert(gatewayEntries.some(({event}) => event === "process.started"))
+  assert(gatewayEntries.some(({event}) => event === "transport.connected"))
+  assert.doesNotMatch(gatewayContents, /process-gateway-token|127\.0\.0\.1|docker-server/)
+})