socketduct 0.1.24 → 0.1.26

package/test/e2e/reverse-client-recovery-process.test.js

@@ -1,12 +1,11 @@
 /* eslint-disable jsdoc/require-returns */
-import assert from "node:assert/strict"
+import {describe, expect, it, withTestCleanup} from "@velocious/testing"
 import {fork} from "node:child_process"
 import {once} from "node:events"
 import {mkdtemp, readFile, rm} from "node:fs/promises"
 import {createServer} from "node:net"
 import {tmpdir} from "node:os"
 import {join} from "node:path"
-import test from "node:test"
 
 import {StaticTokenStore} from "../../src/auth/static-token-store.js"
 import {OutboundReverseGateway, startReverseGatewayHub} from "../../src/reverse/index.js"
@@ -21,7 +20,9 @@ async function listen(server) {
     server.listen(0, "127.0.0.1", () => { server.off("error", reject); resolve(undefined) })
   })
   const address = server.address()
-  assert(address !== null && typeof address !== "string")
+  if (address === null || typeof address === "string") {
+    throw new Error("Server did not bind to a TCP port")
+  }
   return address.port
 }
 
@@ -63,111 +64,114 @@ async function waitForMessage(processState, type) {
   return message
 }
 
-test("SIGKILLed reverse client process recovers its durable stream while hub gateway and target survive", async (t) => {
-  const root = await mkdtemp(join(tmpdir(), "socketduct-reverse-client-process-recovery-"))
-  t.after(() => rm(root, {recursive: true, force: true}))
-  const certificates = generateTestCertificates(root)
-  const spoolDirectory = join(root, "client-spool")
-  let targetAccepts = 0
-  /** @type {import('node:net').Socket | undefined} */
-  let targetSocket
-  const targetSockets = new Set()
-  /** @type {Buffer[]} */
-  const targetReceived = []
-  const target = createServer((socket) => {
-    targetAccepts += 1
-    targetSocket = socket
-    targetSockets.add(socket)
-    socket.once("close", () => targetSockets.delete(socket))
-    socket.on("data", (chunk) => targetReceived.push(Buffer.from(chunk)))
-  })
-  const targetPort = await listen(target)
-  t.after(() => { for (const socket of targetSockets) socket.destroy(); return new Promise((resolve) => target.close(() => resolve(undefined))) })
+describe("reverse client process recovery", () => {
+  it("SIGKILLed reverse client process recovers its durable stream while hub gateway and target survive", async () => withTestCleanup(async (after) => {
+    const root = await mkdtemp(join(tmpdir(), "socketduct-reverse-client-process-recovery-"))
+    after(() => rm(root, {recursive: true, force: true}))
+    const certificates = generateTestCertificates(root)
+    const spoolDirectory = join(root, "client-spool")
+    let targetAccepts = 0
+    /** @type {import('node:net').Socket | undefined} */
+    let targetSocket
+    const targetSockets = new Set()
+    /** @type {Buffer[]} */
+    const targetReceived = []
+    const target = createServer((socket) => {
+      targetAccepts += 1
+      targetSocket = socket
+      targetSockets.add(socket)
+      socket.once("close", () => targetSockets.delete(socket))
+      socket.on("data", (chunk) => targetReceived.push(Buffer.from(chunk)))
+    })
+    const targetPort = await listen(target)
+    after(() => { for (const socket of targetSockets) socket.destroy(); return new Promise((resolve) => target.close(() => resolve(undefined))) })
 
-  const hub = await startReverseGatewayHub({
-    tokens: new StaticTokenStore([{username: "home", token: "gateway-token"}, {username: "app", token: "client-process-token"}]),
-    tls: {certFile: certificates.serverCertFile, keyFile: certificates.serverKeyFile},
-    authorizeGateway: () => true,
-    authorizeClientGateway: () => true,
-    authorizeOpen: () => true,
-    resumeWindowMs: 4_000
-  })
-  t.after(() => hub.close())
-  const gateway = new OutboundReverseGateway({
-    gatewayId: "o2",
-    connectionId: "process_recovery_gateway",
-    hub: {host: "127.0.0.1", port: hub.port, token: "gateway-token"},
-    tls: {caFile: certificates.caCertFile, servername: "localhost"},
-    spoolDirectory: join(root, "gateway-spool"),
-    authorizeTarget: () => true,
-    reconnectDelayMs: 10,
-    maxReconnectDelayMs: 20
-  })
-  t.after(() => gateway.close())
-  await gateway.start()
+    const hub = await startReverseGatewayHub({
+      tokens: new StaticTokenStore([{username: "home", token: "gateway-token"}, {username: "app", token: "client-process-token"}]),
+      tls: {certFile: certificates.serverCertFile, keyFile: certificates.serverKeyFile},
+      authorizeGateway: () => true,
+      authorizeClientGateway: () => true,
+      authorizeOpen: () => true,
+      resumeWindowMs: 4_000
+    })
+    after(() => hub.close())
+    const gateway = new OutboundReverseGateway({
+      gatewayId: "o2",
+      connectionId: "process_recovery_gateway",
+      hub: {host: "127.0.0.1", port: hub.port, token: "gateway-token"},
+      tls: {caFile: certificates.caCertFile, servername: "localhost"},
+      spoolDirectory: join(root, "gateway-spool"),
+      authorizeTarget: () => true,
+      reconnectDelayMs: 10,
+      maxReconnectDelayMs: 20
+    })
+    after(() => gateway.close())
+    await gateway.start()
 
-  const configuration = {
-    gatewayId: "o2",
-    connectionId: "process_recovery_client",
-    hubPort: hub.port,
-    token: "client-process-token",
-    caFile: certificates.caCertFile,
-    spoolDirectory,
-    identifier: "process_recovery_stream",
-    sessionName: "durable-process-stream",
-    targetPort,
-    initialGeneration: 100
-  }
-  const original = startClient({...configuration, phase: "original"})
-  t.after(() => { if (original.child.exitCode === null) original.child.kill("SIGKILL") })
-  const connected = await waitForMessage(original, "connected")
-  original.child.send({type: "write", payload: Buffer.from("before-kill").toString("base64")})
-  await waitForMessage(original, "written")
-  await waitUntil(() => Buffer.concat(targetReceived).toString() === "before-kill", "original target request")
-  assert.equal(targetAccepts, 1)
-  assert(targetSocket !== undefined)
+    const configuration = {
+      gatewayId: "o2",
+      connectionId: "process_recovery_client",
+      hubPort: hub.port,
+      token: "client-process-token",
+      caFile: certificates.caCertFile,
+      spoolDirectory,
+      identifier: "process_recovery_stream",
+      sessionName: "durable-process-stream",
+      targetPort,
+      initialGeneration: 100
+    }
+    const original = startClient({...configuration, phase: "original"})
+    after(() => { if (original.child.exitCode === null) original.child.kill("SIGKILL") })
+    const connected = await waitForMessage(original, "connected")
+    original.child.send({type: "write", payload: Buffer.from("before-kill").toString("base64")})
+    await waitForMessage(original, "written")
+    await waitUntil(() => Buffer.concat(targetReceived).toString() === "before-kill", "original target request")
+    expect(targetAccepts).toBe(1)
+    if (targetSocket === undefined) throw new Error("Expected a live target socket")
 
-  original.child.send({type: "hold-next-data"})
-  await waitForMessage(original, "data-hold-armed")
-  original.child.send({type: "write", payload: Buffer.from("during-kill").toString("base64")})
-  await waitForMessage(original, "data-held")
-  await waitUntil(() => original.messages.filter((message) => message.type === "written").length === 2, "durably spooled held request")
-  assert.equal(Buffer.concat(targetReceived).toString(), "before-kill")
+    original.child.send({type: "hold-next-data"})
+    await waitForMessage(original, "data-hold-armed")
+    original.child.send({type: "write", payload: Buffer.from("during-kill").toString("base64")})
+    await waitForMessage(original, "data-held")
+    await waitUntil(() => original.messages.filter((message) => message.type === "written").length === 2, "durably spooled held request")
+    expect(Buffer.concat(targetReceived).toString()).toBe("before-kill")
 
-  const replayedTargetBytes = Buffer.from("accepted-but-not-durably-acked")
-  original.child.send({type: "observe-buffer", bytes: replayedTargetBytes.length})
-  targetSocket.write(replayedTargetBytes)
-  await waitForMessage(original, "buffered")
-  original.child.send({type: "report-acks"})
-  const acknowledgements = await waitForMessage(original, "acks")
-  assert.equal(acknowledgements.ackFrames, 0, "unread process-local bytes must not be durably acknowledged")
-  await waitUntil(() => gateway.status().spoolBytes === replayedTargetBytes.length, "gateway retains unacknowledged target bytes")
-  const originalExit = once(original.child, "exit")
-  original.child.kill("SIGKILL")
-  await originalExit
+    const replayedTargetBytes = Buffer.from("accepted-but-not-durably-acked")
+    original.child.send({type: "observe-buffer", bytes: replayedTargetBytes.length})
+    targetSocket.write(replayedTargetBytes)
+    await waitForMessage(original, "buffered")
+    original.child.send({type: "report-acks"})
+    const acknowledgements = await waitForMessage(original, "acks")
+    // Unread process-local bytes must not be durably acknowledged.
+    expect(acknowledgements.ackFrames).toBe(0)
+    await waitUntil(() => gateway.status().spoolBytes === replayedTargetBytes.length, "gateway retains unacknowledged target bytes")
+    const originalExit = once(original.child, "exit")
+    original.child.kill("SIGKILL")
+    await originalExit
 
-  const replacement = startClient({...configuration, phase: "replacement"})
-  t.after(() => { if (replacement.child.exitCode === null) replacement.child.kill("SIGKILL") })
-  const recovered = await waitForMessage(replacement, "recovered")
-  const replay = await waitForMessage(replacement, "data")
-  assert.equal(Buffer.from(/** @type {string} */ (replay.payload), "base64").toString(), replayedTargetBytes.toString())
-  assert.equal(recovered.openFrames, 0)
-  assert(/** @type {number} */ (recovered.generation) > /** @type {number} */ (connected.generation))
-  assert.equal(targetAccepts, 1)
-  replacement.child.send({type: "acknowledge", bytes: replayedTargetBytes.length})
-  const acknowledged = await waitForMessage(replacement, "acknowledged")
-  assert.equal(acknowledged.sequence, replayedTargetBytes.length)
-  await waitUntil(() => Buffer.concat(targetReceived).toString() === "before-killduring-kill", "replay strictly above gateway high-water")
+    const replacement = startClient({...configuration, phase: "replacement"})
+    after(() => { if (replacement.child.exitCode === null) replacement.child.kill("SIGKILL") })
+    const recovered = await waitForMessage(replacement, "recovered")
+    const replay = await waitForMessage(replacement, "data")
+    expect(Buffer.from(/** @type {string} */ (replay.payload), "base64").toString()).toBe(replayedTargetBytes.toString())
+    expect(recovered.openFrames).toBe(0)
+    expect(/** @type {number} */ (recovered.generation) > /** @type {number} */ (connected.generation)).toBeTruthy()
+    expect(targetAccepts).toBe(1)
+    replacement.child.send({type: "acknowledge", bytes: replayedTargetBytes.length})
+    const acknowledged = await waitForMessage(replacement, "acknowledged")
+    expect(acknowledged.sequence).toBe(replayedTargetBytes.length)
+    await waitUntil(() => Buffer.concat(targetReceived).toString() === "before-killduring-kill", "replay strictly above gateway high-water")
 
-  replacement.child.send({type: "write", payload: Buffer.from("after-kill").toString("base64")})
-  await waitForMessage(replacement, "written")
-  await waitUntil(() => Buffer.concat(targetReceived).toString() === "before-killduring-killafter-kill", "recovered target request")
-  assert.equal(targetAccepts, 1)
-  const persisted = await readFile(join(spoolDirectory, configuration.identifier, "recovery.json"), "utf8")
-  assert.doesNotMatch(persisted, /client-process-token|gateway-token/)
+    replacement.child.send({type: "write", payload: Buffer.from("after-kill").toString("base64")})
+    await waitForMessage(replacement, "written")
+    await waitUntil(() => Buffer.concat(targetReceived).toString() === "before-killduring-killafter-kill", "recovered target request")
+    expect(targetAccepts).toBe(1)
+    const persisted = await readFile(join(spoolDirectory, configuration.identifier, "recovery.json"), "utf8")
+    expect(persisted).not.toMatch(/client-process-token|gateway-token/)
 
-  const replacementExit = once(replacement.child, "exit")
-  replacement.child.send({type: "close"})
-  await replacementExit
-  assert.equal(replacement.child.exitCode, 0, replacement.stderr())
+    const replacementExit = once(replacement.child, "exit")
+    replacement.child.send({type: "close"})
+    await replacementExit
+    if (replacement.child.exitCode !== 0) throw new Error(`Expected exit code 0: ${replacement.stderr()}`)
+  }))
 })