socketduct 0.1.18 → 0.1.20

package/test/e2e/logger-shutdown-process.test.js

@@ -0,0 +1,86 @@
+/* eslint-disable jsdoc/escape-inline-tags, jsdoc/require-param, jsdoc/require-returns */
+import assert from "node:assert/strict"
+import {spawn} from "node:child_process"
+import {mkdtemp, rm} from "node:fs/promises"
+import {createServer} from "node:net"
+import {tmpdir} from "node:os"
+import {join} from "node:path"
+import test from "node:test"
+
+import {readCompressedLogDirectory} from "../../src/logging/compressed-log-reader.js"
+
+const TIMEOUT_MS = 8_000
+
+test("SIGTERM durably flushes queued critical lifecycle records before exit", async (t) => {
+  const root = await mkdtemp(join(tmpdir(), "socketduct-logger-shutdown-"))
+  const directory = join(root, "logs")
+  const port = await reservePort()
+  const child = spawn(process.execPath, ["src/index.js"], {
+    cwd: process.cwd(),
+    env: {
+      ...process.env,
+      SOCKETDUCT_MODE: "relay",
+      SOCKETDUCT_RELAY_HOST: "127.0.0.1",
+      SOCKETDUCT_RELAY_PORT: String(port),
+      SOCKETDUCT_AUTH_TOKENS: "shutdown-test:shutdown-test-token",
+      SOCKETDUCT_TARGET_ALLOWLIST: "127.0.0.1:1",
+      SOCKETDUCT_STATUS_INTERVAL_MS: "0",
+      SOCKETDUCT_LOG_DIRECTORY: directory,
+      SOCKETDUCT_LOG_FLUSH_INTERVAL_MS: "60000",
+      SOCKETDUCT_LOG_CRITICAL_FLUSH_INTERVAL_MS: "60000"
+    },
+    stdio: ["ignore", "pipe", "pipe"]
+  })
+  let output = ""
+  child.stdout.on("data", (chunk) => { output += chunk.toString() })
+  child.stderr.on("data", (chunk) => { output += chunk.toString() })
+  t.after(async () => {
+    if (child.exitCode === null) child.kill("SIGKILL")
+    await rm(root, {recursive: true, force: true})
+  })
+
+  await waitUntil(() => output.includes('"event":"relay.started"'), child, () => output)
+  assert.equal(child.kill("SIGTERM"), true)
+  const exit = await withTimeout(new Promise((resolve) => child.once("exit", (code, signal) => resolve({code, signal}))), "process exit")
+  assert.deepEqual(exit, {code: 0, signal: null})
+
+  const result = await readCompressedLogDirectory(directory)
+  assert.equal(result.incomplete, false)
+  assert.deepEqual(result.records.map((record) => record.event), ["process.started", "process.stopped", "log_batch_accepted", "log_frame_committed", "log_closed"])
+})
+
+/** @returns {Promise<number>} A released loopback port. */
+async function reservePort() {
+  const server = createServer()
+  await new Promise((resolve, reject) => {
+    server.once("error", reject)
+    server.listen(0, "127.0.0.1", () => { server.off("error", reject); resolve(undefined) })
+  })
+  const address = server.address()
+  assert(address !== null && typeof address !== "string")
+  await new Promise((resolve) => server.close(() => resolve(undefined)))
+  return address.port
+}
+
+/** @param {() => boolean} predicate - Completion check. @param {import('node:child_process').ChildProcess} child - Child process. @param {() => string} output - Captured output. */
+async function waitUntil(predicate, child, output) {
+  if (predicate()) return
+  await withTimeout(new Promise((resolve, reject) => {
+    const onData = () => { if (predicate()) { cleanup(); resolve(undefined) } }
+    const onExit = (/** @type {number | null} */ code) => { cleanup(); reject(new Error(`process exited ${code}: ${output()}`)) }
+    const cleanup = () => { child.stdout?.off("data", onData); child.stderr?.off("data", onData); child.off("exit", onExit) }
+    child.stdout?.on("data", onData)
+    child.stderr?.on("data", onData)
+    child.once("exit", onExit)
+  }), "process startup")
+}
+
+/** @template T @param {Promise<T>} promise - Work. @param {string} label - Timeout label. @returns {Promise<T>} Result. */
+async function withTimeout(promise, label) {
+  let timer
+  try {
+    return await Promise.race([promise, new Promise((_, reject) => { timer = setTimeout(() => reject(new Error(`Timed out: ${label}`)), TIMEOUT_MS) })])
+  } finally {
+    if (timer !== undefined) clearTimeout(timer)
+  }
+}

package/test/e2e/reverse-client-recovery-process.test.js

@@ -0,0 +1,170 @@
+/* eslint-disable jsdoc/require-returns */
+import assert from "node:assert/strict"
+import {fork} from "node:child_process"
+import {once} from "node:events"
+import {mkdtemp, readFile, rm} from "node:fs/promises"
+import {createServer} from "node:net"
+import {tmpdir} from "node:os"
+import {join} from "node:path"
+import test from "node:test"
+
+import {StaticTokenStore} from "../../src/auth/static-token-store.js"
+import {OutboundReverseGateway, startReverseGatewayHub} from "../../src/reverse/index.js"
+import {generateTestCertificates} from "../helpers/tls-certificates.js"
+
+/** @typedef {{gatewayId: string, connectionId: string, hubPort: number, token: string, caFile: string, spoolDirectory: string, identifier: string, sessionName: string, targetPort: number, initialGeneration: number, phase: string}} RecoveryFixtureConfiguration */
+
+/** @param {import('node:net').Server} server - Server. */
+async function listen(server) {
+  await new Promise((resolve, reject) => {
+    server.once("error", reject)
+    server.listen(0, "127.0.0.1", () => { server.off("error", reject); resolve(undefined) })
+  })
+  const address = server.address()
+  assert(address !== null && typeof address !== "string")
+  return address.port
+}
+
+/**
+ * @param {() => boolean} predicate - Completion predicate.
+ * @param {string} label - Timeout label.
+ */
+async function waitUntil(predicate, label) {
+  const deadline = Date.now() + 8_000
+  while (!predicate()) {
+    if (Date.now() >= deadline) throw new Error(`Timed out: ${label}`)
+    await new Promise((resolve) => setTimeout(resolve, 10))
+  }
+}
+
+/** @param {RecoveryFixtureConfiguration} configuration - Child configuration. */
+function startClient(configuration) {
+  const child = fork("test/fixtures/reverse-client-recovery.mjs", [], {
+    cwd: process.cwd(),
+    env: {...process.env, SOCKETDUCT_RECOVERY_FIXTURE: JSON.stringify(configuration)},
+    stdio: ["ignore", "pipe", "pipe", "ipc"]
+  })
+  /** @type {Record<string, unknown>[]} */
+  const messages = []
+  let stderr = ""
+  child.on("message", (message) => messages.push(/** @type {Record<string, unknown>} */ (message)))
+  child.stderr?.on("data", (chunk) => { stderr += chunk.toString() })
+  return {child, messages, stderr: () => stderr}
+}
+
+/**
+ * @param {ReturnType<typeof startClient>} processState - Child and captured messages.
+ * @param {string} type - Expected message type.
+ */
+async function waitForMessage(processState, type) {
+  await waitUntil(() => processState.messages.some((message) => message.type === type) || processState.child.exitCode !== null, `child message ${type}`)
+  const message = processState.messages.find((candidate) => candidate.type === type)
+  if (message === undefined) throw new Error(`Child exited before ${type}: ${processState.stderr()}`)
+  return message
+}
+
+test("SIGKILLed reverse client process recovers its durable stream while hub gateway and target survive", async (t) => {
+  const root = await mkdtemp(join(tmpdir(), "socketduct-reverse-client-process-recovery-"))
+  t.after(() => rm(root, {recursive: true, force: true}))
+  const certificates = generateTestCertificates(root)
+  const spoolDirectory = join(root, "client-spool")
+  let targetAccepts = 0
+  /** @type {import('node:net').Socket | undefined} */
+  let targetSocket
+  const targetSockets = new Set()
+  /** @type {Buffer[]} */
+  const targetReceived = []
+  const target = createServer((socket) => {
+    targetAccepts += 1
+    targetSocket = socket
+    targetSockets.add(socket)
+    socket.once("close", () => targetSockets.delete(socket))
+    socket.on("data", (chunk) => targetReceived.push(Buffer.from(chunk)))
+  })
+  const targetPort = await listen(target)
+  t.after(() => { for (const socket of targetSockets) socket.destroy(); return new Promise((resolve) => target.close(() => resolve(undefined))) })
+
+  const hub = await startReverseGatewayHub({
+    tokens: new StaticTokenStore([{username: "home", token: "gateway-token"}, {username: "app", token: "client-process-token"}]),
+    tls: {certFile: certificates.serverCertFile, keyFile: certificates.serverKeyFile},
+    authorizeGateway: () => true,
+    authorizeClientGateway: () => true,
+    authorizeOpen: () => true,
+    resumeWindowMs: 4_000
+  })
+  t.after(() => hub.close())
+  const gateway = new OutboundReverseGateway({
+    gatewayId: "o2",
+    connectionId: "process_recovery_gateway",
+    hub: {host: "127.0.0.1", port: hub.port, token: "gateway-token"},
+    tls: {caFile: certificates.caCertFile, servername: "localhost"},
+    spoolDirectory: join(root, "gateway-spool"),
+    authorizeTarget: () => true,
+    reconnectDelayMs: 10,
+    maxReconnectDelayMs: 20
+  })
+  t.after(() => gateway.close())
+  await gateway.start()
+
+  const configuration = {
+    gatewayId: "o2",
+    connectionId: "process_recovery_client",
+    hubPort: hub.port,
+    token: "client-process-token",
+    caFile: certificates.caCertFile,
+    spoolDirectory,
+    identifier: "process_recovery_stream",
+    sessionName: "durable-process-stream",
+    targetPort,
+    initialGeneration: 100
+  }
+  const original = startClient({...configuration, phase: "original"})
+  t.after(() => { if (original.child.exitCode === null) original.child.kill("SIGKILL") })
+  const connected = await waitForMessage(original, "connected")
+  original.child.send({type: "write", payload: Buffer.from("before-kill").toString("base64")})
+  await waitForMessage(original, "written")
+  await waitUntil(() => Buffer.concat(targetReceived).toString() === "before-kill", "original target request")
+  assert.equal(targetAccepts, 1)
+  assert(targetSocket !== undefined)
+
+  original.child.send({type: "hold-next-data"})
+  await waitForMessage(original, "data-hold-armed")
+  original.child.send({type: "write", payload: Buffer.from("during-kill").toString("base64")})
+  await waitForMessage(original, "data-held")
+  await waitUntil(() => original.messages.filter((message) => message.type === "written").length === 2, "durably spooled held request")
+  assert.equal(Buffer.concat(targetReceived).toString(), "before-kill")
+
+  const replayedTargetBytes = Buffer.from("accepted-but-not-durably-acked")
+  original.child.send({type: "observe-buffer", bytes: replayedTargetBytes.length})
+  targetSocket.write(replayedTargetBytes)
+  await waitForMessage(original, "buffered")
+  original.child.send({type: "report-acks"})
+  const acknowledgements = await waitForMessage(original, "acks")
+  assert.equal(acknowledgements.ackFrames, 0, "unread process-local bytes must not be durably acknowledged")
+  await waitUntil(() => gateway.status().spoolBytes === replayedTargetBytes.length, "gateway retains unacknowledged target bytes")
+  const originalExit = once(original.child, "exit")
+  original.child.kill("SIGKILL")
+  await originalExit
+
+  const replacement = startClient({...configuration, phase: "replacement"})
+  t.after(() => { if (replacement.child.exitCode === null) replacement.child.kill("SIGKILL") })
+  const recovered = await waitForMessage(replacement, "recovered")
+  const replay = await waitForMessage(replacement, "data")
+  assert.equal(Buffer.from(/** @type {string} */ (replay.payload), "base64").toString(), replayedTargetBytes.toString())
+  assert.equal(recovered.openFrames, 0)
+  assert(/** @type {number} */ (recovered.generation) > /** @type {number} */ (connected.generation))
+  assert.equal(targetAccepts, 1)
+  await waitUntil(() => Buffer.concat(targetReceived).toString() === "before-killduring-kill", "replay strictly above gateway high-water")
+
+  replacement.child.send({type: "write", payload: Buffer.from("after-kill").toString("base64")})
+  await waitForMessage(replacement, "written")
+  await waitUntil(() => Buffer.concat(targetReceived).toString() === "before-killduring-killafter-kill", "recovered target request")
+  assert.equal(targetAccepts, 1)
+  const persisted = await readFile(join(spoolDirectory, configuration.identifier, "recovery.json"), "utf8")
+  assert.doesNotMatch(persisted, /client-process-token|gateway-token/)
+
+  const replacementExit = once(replacement.child, "exit")
+  replacement.child.send({type: "close"})
+  await replacementExit
+  assert.equal(replacement.child.exitCode, 0, replacement.stderr())
+})

package/test/e2e/reverse-gateway-process.test.js

@@ -1,7 +1,7 @@
 /* eslint-disable jsdoc/escape-inline-tags, jsdoc/require-param, jsdoc/require-returns */
 import assert from "node:assert/strict"
 import {spawn} from "node:child_process"
-import {mkdtemp, readFile, readdir, rm} from "node:fs/promises"
+import {mkdtemp, rm} from "node:fs/promises"
 import {createServer} from "node:net"
 import {tmpdir} from "node:os"
 import {join} from "node:path"
@@ -9,10 +9,12 @@ import test from "node:test"
 import {connect as connectTls} from "node:tls"
 
 import {ReverseGatewayConnection} from "../../src/reverse/client.js"
+import {readCompressedLogDirectory} from "../../src/logging/compressed-log-reader.js"
 import {generateTestCertificates} from "../helpers/tls-certificates.js"
 
 const PROCESS_TIMEOUT_MS = 8_000
 const PROCESS_LISTEN_ATTEMPTS = 3
+const TEST_FINGERPRINT_KEY = Buffer.alloc(32, 0x5a).toString("base64url")
 
 /** A child exit observed before its expected startup event. */
 class ProcessStartupExitError extends Error {
@@ -234,8 +236,9 @@ test("spawned reverse hub and outbound gateway expose an allowlisted Docker stre
     SOCKETDUCT_REVERSE_ROUTES: `tensorbuzz:o2:127.0.0.1:${targetPort}`,
     SOCKETDUCT_STATUS_INTERVAL_MS: "0",
     SOCKETDUCT_LOG_DIRECTORY: logDirectory,
-    SOCKETDUCT_LOG_MAX_FILE_BYTES: "4096",
-    SOCKETDUCT_LOG_MAX_FILES: "2"
+    SOCKETDUCT_LOG_SEGMENT_MAX_BYTES: "4096",
+    SOCKETDUCT_CONNECTION_FINGERPRINT_KEY: TEST_FINGERPRINT_KEY,
+    SOCKETDUCT_CONNECTION_FINGERPRINT_KEY_EPOCH: "process-test"
   }, "reverse-hub.started")
   t.after(() => stopProcess(hub))
   const gateway = await startProcess({
@@ -250,8 +253,7 @@ test("spawned reverse hub and outbound gateway expose an allowlisted Docker stre
     SOCKETDUCT_TARGET_ALLOWLIST: `127.0.0.1:${targetPort}`,
     SOCKETDUCT_STATUS_INTERVAL_MS: "0",
     SOCKETDUCT_LOG_DIRECTORY: logDirectory,
-    SOCKETDUCT_LOG_MAX_FILE_BYTES: "4096",
-    SOCKETDUCT_LOG_MAX_FILES: "2"
+    SOCKETDUCT_LOG_SEGMENT_MAX_BYTES: "4096"
   }, "reverse-gateway.started")
   t.after(() => stopProcess(gateway))
 
@@ -270,19 +272,18 @@ test("spawned reverse hub and outbound gateway expose an allowlisted Docker stre
   assert.equal(Buffer.from(/** @type {Uint8Array} */ (await withTimeout(response, "process response"))).toString(), "docker:GET /version")
   assert.equal(client.status().compression, "zstd")
   socket.destroy()
+  client.close()
+  await stopProcess(gateway)
+  await stopProcess(hub)
 
-  assert(hub.pid !== undefined)
-  assert(gateway.pid !== undefined)
-  const files = (await readdir(logDirectory)).sort()
-  assert(files.includes(`reverse-hub-${hub.pid}.jsonl`))
-  assert(files.includes(`reverse-gateway-${gateway.pid}.jsonl`))
-  const hubEntries = (await readFile(join(logDirectory, `reverse-hub-${hub.pid}.jsonl`), "utf8")).trim().split("\n").map((line) => JSON.parse(line))
-  const gatewayContents = await readFile(join(logDirectory, `reverse-gateway-${gateway.pid}.jsonl`), "utf8")
-  const gatewayEntries = gatewayContents.trim().split("\n").map((line) => JSON.parse(line))
+  const logs = await readCompressedLogDirectory(logDirectory)
+  assert.equal(logs.incomplete, false)
+  const hubEntries = logs.records.filter(({role}) => role === "reverse-hub")
+  const gatewayEntries = logs.records.filter(({role}) => role === "reverse-gateway")
   assert(hubEntries.some(({event}) => event === "process.started"))
   assert(gatewayEntries.some(({event}) => event === "process.started"))
   assert(gatewayEntries.some(({event}) => event === "transport.connected"))
-  assert.doesNotMatch(gatewayContents, /process-gateway-token|127\.0\.0\.1|docker-server/)
+  assert.doesNotMatch(JSON.stringify(logs.records), /process-gateway-token|127\.0\.0\.1|docker-server|process_home_o2_gateway/)
 })
 
 test("one route-class client physical connection interleaves streams through different gateway processes", async (t) => {

package/test/fixtures/reverse-client-recovery.mjs

@@ -0,0 +1,123 @@
+import {connect as connectTls} from "node:tls"
+
+import {V3FrameType} from "../../src/protocol/v3-frame.js"
+import {ReverseGatewayConnection} from "../../src/reverse/client.js"
+
+const configuration = JSON.parse(process.env.SOCKETDUCT_RECOVERY_FIXTURE ?? "null")
+if (configuration === null) throw new Error("Missing recovery fixture configuration")
+
+let openFrames = 0
+let ackFrames = 0
+let holdNextData = false
+const client = new ReverseGatewayConnection({
+  gatewayId: configuration.gatewayId,
+  connectionId: configuration.connectionId,
+  hub: {host: "127.0.0.1", port: configuration.hubPort, token: configuration.token},
+  tls: {caFile: configuration.caFile, servername: "localhost"},
+  spoolDirectory: configuration.spoolDirectory,
+  initialGeneration: configuration.phase === "replacement" ? 0 : configuration.initialGeneration,
+  reconnectDelayMs: 10,
+  maxReconnectDelayMs: 20,
+  createPhysicalSocket: (options) => {
+    const physical = connectTls({host: options.host, port: options.port, ...options.tls})
+    const write = physical.write.bind(physical)
+    physical.write = ((chunk, ...args) => {
+      if (Buffer.isBuffer(chunk) && chunk.length >= 6 && chunk[5] === V3FrameType.OPEN) openFrames += 1
+      if (Buffer.isBuffer(chunk) && chunk.length >= 6 && chunk[5] === V3FrameType.ACK) ackFrames += 1
+      if (holdNextData && Buffer.isBuffer(chunk) && chunk.length >= 6 && chunk[5] === V3FrameType.DATA) {
+        holdNextData = false
+        report("data-held")
+        return true
+      }
+      return Reflect.apply(write, physical, [chunk, ...args])
+    })
+    return physical
+  }
+})
+
+const streamOptions = {
+  recover: true,
+  identifier: configuration.identifier,
+  sessionName: configuration.sessionName,
+  host: "127.0.0.1",
+  port: configuration.targetPort
+}
+const socket = configuration.phase === "replacement"
+  ? client.registerRecoveredConnection(streamOptions)
+  : undefined
+
+/**
+ * @param {string} type - Parent message type.
+ * @param {Record<string, unknown>} [fields] - Message fields.
+ */
+function report(type, fields = {}) {
+  process.send?.({type, ...fields})
+}
+
+/** @param {import('../../src/client/virtual-socket.js').VirtualSocket} activeSocket - Stream under test. */
+function installSocketEvents(activeSocket) {
+  activeSocket.once("connect", () => report(configuration.phase === "replacement" ? "recovered" : "connected", {
+    generation: client.status().connectionGeneration,
+    openFrames
+  }))
+  activeSocket.on("error", (error) => report("stream-error", {code: error.code ?? "ERROR"}))
+}
+
+let activeSocket = socket
+if (activeSocket !== undefined) {
+  installSocketEvents(activeSocket)
+  activeSocket.on("data", (chunk) => report("data", {payload: Buffer.from(chunk).toString("base64")}))
+  await client.connect()
+} else {
+  await client.connect()
+  activeSocket = client.createConnection({
+    identifier: configuration.identifier,
+    sessionName: configuration.sessionName,
+    host: "127.0.0.1",
+    port: configuration.targetPort
+  })
+  installSocketEvents(activeSocket)
+}
+
+process.on("message", (message) => {
+  if (message?.type === "hold-next-data") {
+    holdNextData = true
+    report("data-hold-armed")
+    return
+  }
+  if (message?.type === "write") {
+    activeSocket.write(Buffer.from(message.payload, "base64"), (error) => report("written", {code: error?.code ?? null}))
+    return
+  }
+  if (message?.type === "observe-buffer") {
+    const timer = setInterval(() => {
+      if (activeSocket.readableLength < message.bytes) return
+      clearInterval(timer)
+      report("buffered", {bytes: activeSocket.readableLength})
+    }, 5)
+    timer.unref?.()
+    return
+  }
+  if (message?.type === "report-acks") {
+    setTimeout(() => report("acks", {ackFrames}), 50)
+    return
+  }
+  if (message?.type === "close") {
+    activeSocket.destroy()
+    void client.close().then(() => process.exit(0), (error) => {
+      report("close-error", {code: error?.code ?? "ERROR"})
+      process.exit(1)
+    })
+  }
+})
+
+process.on("uncaughtException", (error) => {
+  report("fatal", {message: error.message, code: error.code ?? "ERROR"})
+  process.exit(1)
+})
+
+process.on("unhandledRejection", (error) => {
+  const failure = error instanceof Error ? error : new Error(String(error))
+  report("fatal", {message: failure.message, code: failure.code ?? "ERROR"})
+  process.exit(1)
+})

package/test/integration/logging-cli.test.js

@@ -0,0 +1,168 @@
+/* eslint-disable jsdoc/escape-inline-tags, jsdoc/require-param, jsdoc/require-returns */
+import assert from "node:assert/strict"
+import {execFile, spawn} from "node:child_process"
+import {mkdtemp, readFile, readdir, rm, utimes, writeFile} from "node:fs/promises"
+import {tmpdir} from "node:os"
+import {join} from "node:path"
+import test from "node:test"
+import {promisify} from "node:util"
+
+import {createCompressedJsonlFileLogger} from "../../src/logging/compressed-log-writer.js"
+import {CompressedLogTailReader} from "../../src/logging/compressed-log-reader.js"
+import {enforceLogRetention} from "../../src/logging/log-retention.js"
+
+const run = promisify(execFile)
+const cli = new URL("../../src/cli/socketduct.js", import.meta.url)
+
+test("owned CLI searches and tails durable compressed records with stable exits", async (t) => {
+  const root = await mkdtemp(join(tmpdir(), "socketduct-logs-cli-"))
+  const directory = join(root, "logs")
+  t.after(() => rm(root, {recursive: true, force: true}))
+  const logger = createCompressedJsonlFileLogger({directory, role: "reverse-hub"})
+  logger.log("process.started", {runtimeMode: "reverse-hub"})
+  logger.log("runtime.status", {virtualStreams: 7})
+  await logger.close()
+
+  const match = await invoke("logs", "search", "--directory", directory, "--event", "runtime.status")
+  assert.equal(match.code, 0)
+  assert.deepEqual(match.stdout.trim().split("\n").map((line) => JSON.parse(line).event), ["runtime.status"])
+
+  const absent = await invoke("logs", "search", "--directory", directory, "--event", "transport.failed")
+  assert.equal(absent.code, 1)
+  assert.equal(absent.stdout, "")
+
+  const tail = await invoke("logs", "tail", "--directory", directory)
+  assert.equal(tail.code, 0)
+  assert.deepEqual(tail.stdout.trim().split("\n").map((line) => JSON.parse(line).event), ["process.started", "runtime.status", "log_batch_accepted", "log_frame_committed", "log_closed"])
+
+  const invalid = await invoke("logs", "search", "--directory", directory, "--client-generation", "not-a-number")
+  assert.equal(invalid.code, 2)
+  assert.match(invalid.stderr, /Socketduct logs error/)
+})
+
+test("owned CLI follow stays live and emits newly durable writer records", async (t) => {
+  const root = await mkdtemp(join(tmpdir(), "socketduct-logs-follow-"))
+  const directory = join(root, "logs")
+  t.after(() => rm(root, {recursive: true, force: true}))
+  const initial = createCompressedJsonlFileLogger({directory, role: "reverse-client"})
+  initial.log("process.started", {runtimeMode: "reverse-client"})
+  await initial.close()
+
+  const child = spawn(process.execPath, [cli.pathname, "logs", "tail", "--directory", directory, "--follow", "--poll-ms", "10"], {stdio: ["ignore", "pipe", "pipe"]})
+  let stdout = ""
+  let stderr = ""
+  child.stdout.on("data", (chunk) => { stdout += chunk.toString() })
+  child.stderr.on("data", (chunk) => { stderr += chunk.toString() })
+  t.after(() => { if (child.exitCode === null) child.kill("SIGKILL") })
+  await waitFor(() => stdout.includes('"eventSequence":"1"'), child, () => stderr)
+
+  const later = createCompressedJsonlFileLogger({directory, role: "reverse-client"})
+  later.log("runtime.status", {virtualStreams: 2})
+  await later.close()
+  await waitFor(() => stdout.includes('"event":"runtime.status"'), child, () => stderr)
+  child.kill("SIGTERM")
+  const code = await new Promise((resolve) => child.once("exit", resolve))
+  assert.equal(code, 0)
+})
+
+test("owned CLI exits 2 instead of hiding corrupt interior evidence", async (t) => {
+  const root = await mkdtemp(join(tmpdir(), "socketduct-logs-corrupt-"))
+  const directory = join(root, "logs")
+  t.after(() => rm(root, {recursive: true, force: true}))
+  const logger = createCompressedJsonlFileLogger({directory, role: "reverse-gateway"})
+  logger.log("process.started", {runtimeMode: "reverse-gateway"})
+  await logger.close()
+  const name = (await readdir(directory)).find((entry) => entry.endsWith(".jsonl.zst"))
+  assert(name !== undefined)
+  const path = join(directory, name)
+  const bytes = await readFile(path)
+  bytes[bytes.length - 1] = /** @type {number} */ (bytes[bytes.length - 1]) ^ 0xff
+  await writeFile(path, bytes)
+
+  const result = await invoke("logs", "search", "--directory", directory, "--event", "process.started")
+  assert.equal(result.code, 2)
+  assert.equal(result.stdout, "")
+  assert.match(result.stderr, /Socketduct log evidence error/)
+})
+
+test("tail reader incrementally follows rotation and prunes retention-deleted cursors", async (t) => {
+  const root = await mkdtemp(join(tmpdir(), "socketduct-logs-incremental-tail-"))
+  const directory = join(root, "logs")
+  t.after(() => rm(root, {recursive: true, force: true}))
+  const logger = createCompressedJsonlFileLogger({directory, role: "reverse-hub", frameMaxBytes: 256, maxSegmentBytes: 1024, compressionLevel: 1})
+  await logger.flush()
+  for (let index = 0; index < 8; index += 1) {
+    logger.log("process.started", {runtimeMode: "reverse-hub"})
+    await logger.flush()
+  }
+  await logger.flush()
+
+  const reader = new CompressedLogTailReader(directory)
+  const initial = await reader.poll()
+  assert(initial.records.length > 0)
+  const identities = new Set(initial.records.map((record) => `${record.writerEpoch}\0${record.eventSequence}`))
+  assert.equal(identities.size, initial.records.length)
+  const decoded = reader.status().decodedFrames
+  assert(decoded > 0)
+  assert.deepEqual((await reader.poll()).records, [])
+  assert.equal(reader.status().decodedFrames, decoded)
+
+  logger.log("process.started", {runtimeMode: "reverse-hub"})
+  await logger.flush()
+  await logger.flush()
+  const activeDelta = await reader.poll()
+  assert(activeDelta.records.length > 0)
+  for (const record of activeDelta.records) assert.equal(identities.has(`${record.writerEpoch}\0${record.eventSequence}`), false)
+  await logger.close()
+  const closeDelta = await reader.poll()
+  assert(closeDelta.records.length > 0)
+  for (const record of closeDelta.records) assert.equal(identities.has(`${record.writerEpoch}\0${record.eventSequence}`), false)
+
+  const oldSegments = (await readdir(directory)).filter((name) => name.endsWith(".jsonl.zst"))
+  assert(oldSegments.length > 1, "fixture must rotate through more than one segment")
+  for (const name of oldSegments) await utimes(join(directory, name), new Date(1_000), new Date(1_000))
+  const later = createCompressedJsonlFileLogger({directory, role: "reverse-hub", compressionLevel: 1})
+  later.log("runtime.status", {virtualStreams: 2})
+  await later.close()
+  const laterSegments = (await readdir(directory)).filter((name) => name.endsWith(".jsonl.zst") && !oldSegments.includes(name))
+  for (const name of laterSegments) await utimes(join(directory, name), new Date(10_000), new Date(10_000))
+  const laterDelta = await reader.poll()
+  assert(laterDelta.records.some((record) => record.event === "runtime.status"))
+
+  await enforceLogRetention({directory, maxAgeMs: 5_000, maxBytes: 1_000_000, now: () => new Date(10_000)})
+  const beforePruneDecode = reader.status().decodedFrames
+  assert.deepEqual((await reader.poll()).records, [])
+  const survivingSegments = (await readdir(directory)).filter((name) => name.endsWith(".jsonl.zst")).length
+  assert(reader.status().trackedSegments <= survivingSegments)
+  assert.equal(reader.status().decodedFrames, beforePruneDecode)
+})
+
+/** @param {...string} args - CLI arguments. */
+async function invoke(...args) {
+  try {
+    const result = await run(process.execPath, [cli.pathname, ...args])
+    return {code: 0, stdout: result.stdout, stderr: result.stderr}
+  } catch (error) {
+    if (error === null || typeof error !== "object") throw error
+    return {
+      code: /** @type {{code?: number}} */ (error).code,
+      stdout: /** @type {{stdout?: string}} */ (error).stdout ?? "",
+      stderr: /** @type {{stderr?: string}} */ (error).stderr ?? ""
+    }
+  }
+}
+
+/** @param {() => boolean} predicate - Completion check. @param {import('node:child_process').ChildProcess} child - CLI process. @param {() => string} stderr - Error output. */
+async function waitFor(predicate, child, stderr) {
+  if (predicate()) return
+  /** @type {ReturnType<typeof setTimeout> | undefined} */
+  let timer
+  await new Promise((resolve, reject) => {
+    const cleanup = () => { if (timer !== undefined) clearTimeout(timer); child.stdout?.off("data", onData); child.off("exit", onExit) }
+    const onData = () => { if (predicate()) { cleanup(); resolve(undefined) } }
+    const onExit = (/** @type {number | null} */ code) => { cleanup(); reject(new Error(`CLI exited ${code}: ${stderr()}`)) }
+    timer = setTimeout(() => { cleanup(); reject(new Error(`Timed out waiting for CLI output: ${stderr()}`)) }, 5_000)
+    child.stdout?.on("data", onData)
+    child.once("exit", onExit)
+  })
+}