socketduct 0.1.25 → 0.1.26

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/AGENTS.md +2 -2
  2. package/changelog.d/20261010130509-bound-log-frame-compression-memory.md +1 -0
  3. package/docs/runbooks/jsonl-integrity-rollout.md +8 -6
  4. package/package.json +2 -2
  5. package/scripts/test.js +9 -14
  6. package/src/logging/compressed-log-writer.js +1 -1
  7. package/src/logging/zstd-frame-codec.js +15 -1
  8. package/test/e2e/gateway-process.test.js +93 -92
  9. package/test/e2e/logger-shutdown-process.test.js +40 -37
  10. package/test/e2e/reverse-client-recovery-process.test.js +107 -103
  11. package/test/e2e/reverse-gateway-process.test.js +328 -318
  12. package/test/integration/initial-target-recovery.test.js +333 -334
  13. package/test/integration/logging-cli.test.js +121 -119
  14. package/test/integration/logging-compressed-recovery.test.js +96 -95
  15. package/test/integration/multiplexed-compatibility.test.js +178 -177
  16. package/test/integration/multiplexed-corrections.test.js +360 -359
  17. package/test/integration/multiplexed-failures.test.js +253 -251
  18. package/test/integration/multiplexed-transport-set.test.js +334 -322
  19. package/test/integration/multiplexed-transport.test.js +135 -134
  20. package/test/integration/reverse-gateway-stream-id-reset.test.js +90 -89
  21. package/test/integration/reverse-gateway.test.js +2309 -2218
  22. package/test/integration/reverse-resume-continuity.test.js +219 -218
  23. package/test/integration/reverse-route-class.test.js +358 -341
  24. package/test/integration/socketduct-http-agent.test.js +314 -312
  25. package/test/integration/socketduct-https-agent.test.js +516 -485
  26. package/test/integration/stream-compression.test.js +862 -824
  27. package/test/integration/tcp-forwarding.test.js +2550 -2447
  28. package/test/integration/tls-forwarding.test.js +593 -592
  29. package/test/unit/auth/auth.test.js +43 -44
  30. package/test/unit/auth/static-token-store.test.js +9 -8
  31. package/test/unit/client/connect.test.js +41 -40
  32. package/test/unit/client/deferred-virtual-socket.test.js +95 -94
  33. package/test/unit/client/multiplexed-spool-budget.test.js +155 -154
  34. package/test/unit/client/multiplexed-transport-set.test.js +830 -748
  35. package/test/unit/client/socketduct-socket.test.js +768 -771
  36. package/test/unit/client/virtual-socket.test.js +198 -168
  37. package/test/unit/config/docker-gateway-files.test.js +224 -224
  38. package/test/unit/config/environment.test.js +472 -492
  39. package/test/unit/index.test.js +53 -52
  40. package/test/unit/logging/compressed-log-worker.test.js +23 -22
  41. package/test/unit/logging/compressed-log-writer.test.js +155 -154
  42. package/test/unit/logging/log-retention.test.js +52 -47
  43. package/test/unit/logging/log-segment-index.test.js +84 -81
  44. package/test/unit/logging/zstd-frame-codec.test.js +28 -19
  45. package/test/unit/protocol/compression.test.js +254 -298
  46. package/test/unit/protocol/frame.test.js +280 -276
  47. package/test/unit/protocol/v3-compression.test.js +21 -23
  48. package/test/unit/protocol/v3-frame.test.js +42 -41
  49. package/test/unit/release-configuration.test.js +15 -12
  50. package/test/unit/reverse/disconnect-attribution.test.js +65 -64
  51. package/test/unit/reverse-terminal-outcomes.test.js +53 -50
  52. package/test/unit/server/control-handler.test.js +232 -231
  53. package/test/unit/server/primary.test.js +51 -49
  54. package/test/unit/server/relay-runtime.test.js +280 -294
  55. package/test/unit/sessions/session.test.js +48 -62
  56. package/test/unit/spool/spool-file.test.js +401 -385
@@ -1,10 +1,9 @@
1
1
  /* eslint-disable jsdoc/escape-inline-tags, jsdoc/require-param, jsdoc/require-returns */
2
- import assert from "node:assert/strict"
2
+ import {describe, expect, it, withTestCleanup} from "@velocious/testing"
3
3
  import {execFile, spawn} from "node:child_process"
4
4
  import {mkdtemp, readFile, readdir, rm, utimes, writeFile} from "node:fs/promises"
5
5
  import {tmpdir} from "node:os"
6
6
  import {join} from "node:path"
7
- import test from "node:test"
8
7
  import {promisify} from "node:util"
9
8
 
10
9
  import {createCompressedJsonlFileLogger} from "../../src/logging/compressed-log-writer.js"
@@ -14,127 +13,130 @@ import {enforceLogRetention} from "../../src/logging/log-retention.js"
14
13
  const run = promisify(execFile)
15
14
  const cli = new URL("../../src/cli/socketduct.js", import.meta.url)
16
15
 
17
- test("owned CLI searches and tails durable compressed records with stable exits", async (t) => {
18
- const root = await mkdtemp(join(tmpdir(), "socketduct-logs-cli-"))
19
- const directory = join(root, "logs")
20
- t.after(() => rm(root, {recursive: true, force: true}))
21
- const logger = createCompressedJsonlFileLogger({directory, role: "reverse-hub"})
22
- logger.log("process.started", {runtimeMode: "reverse-hub"})
23
- logger.log("runtime.status", {virtualStreams: 7})
24
- await logger.close()
25
-
26
- const match = await invoke("logs", "search", "--directory", directory, "--event", "runtime.status")
27
- assert.equal(match.code, 0)
28
- assert.deepEqual(match.stdout.trim().split("\n").map((line) => JSON.parse(line).event), ["runtime.status"])
29
-
30
- const absent = await invoke("logs", "search", "--directory", directory, "--event", "transport.failed")
31
- assert.equal(absent.code, 1)
32
- assert.equal(absent.stdout, "")
33
-
34
- const tail = await invoke("logs", "tail", "--directory", directory)
35
- assert.equal(tail.code, 0)
36
- assert.deepEqual(tail.stdout.trim().split("\n").map((line) => JSON.parse(line).event), ["process.started", "runtime.status", "log_batch_accepted", "log_frame_committed", "log_closed"])
37
-
38
- const invalid = await invoke("logs", "search", "--directory", directory, "--client-generation", "not-a-number")
39
- assert.equal(invalid.code, 2)
40
- assert.match(invalid.stderr, /Socketduct logs error/)
41
- })
42
-
43
- test("owned CLI follow stays live and emits newly durable writer records", async (t) => {
44
- const root = await mkdtemp(join(tmpdir(), "socketduct-logs-follow-"))
45
- const directory = join(root, "logs")
46
- t.after(() => rm(root, {recursive: true, force: true}))
47
- const initial = createCompressedJsonlFileLogger({directory, role: "reverse-client"})
48
- initial.log("process.started", {runtimeMode: "reverse-client"})
49
- await initial.close()
50
-
51
- const child = spawn(process.execPath, [cli.pathname, "logs", "tail", "--directory", directory, "--follow", "--poll-ms", "10"], {stdio: ["ignore", "pipe", "pipe"]})
52
- let stdout = ""
53
- let stderr = ""
54
- child.stdout.on("data", (chunk) => { stdout += chunk.toString() })
55
- child.stderr.on("data", (chunk) => { stderr += chunk.toString() })
56
- t.after(() => { if (child.exitCode === null) child.kill("SIGKILL") })
57
- await waitFor(() => stdout.includes('"eventSequence":"1"'), child, () => stderr)
58
-
59
- const later = createCompressedJsonlFileLogger({directory, role: "reverse-client"})
60
- later.log("runtime.status", {virtualStreams: 2})
61
- await later.close()
62
- await waitFor(() => stdout.includes('"event":"runtime.status"'), child, () => stderr)
63
- child.kill("SIGTERM")
64
- const code = await new Promise((resolve) => child.once("exit", resolve))
65
- assert.equal(code, 0)
66
- })
16
+ describe("socketduct logs CLI", () => {
17
+ it("owned CLI searches and tails durable compressed records with stable exits", async () => withTestCleanup(async (after) => {
18
+ const root = await mkdtemp(join(tmpdir(), "socketduct-logs-cli-"))
19
+ const directory = join(root, "logs")
20
+ after(() => rm(root, {recursive: true, force: true}))
21
+ const logger = createCompressedJsonlFileLogger({directory, role: "reverse-hub"})
22
+ logger.log("process.started", {runtimeMode: "reverse-hub"})
23
+ logger.log("runtime.status", {virtualStreams: 7})
24
+ await logger.close()
25
+
26
+ const match = await invoke("logs", "search", "--directory", directory, "--event", "runtime.status")
27
+ expect(match.code).toBe(0)
28
+ expect(match.stdout.trim().split("\n").map((line) => JSON.parse(line).event)).toEqual(["runtime.status"])
29
+
30
+ const absent = await invoke("logs", "search", "--directory", directory, "--event", "transport.failed")
31
+ expect(absent.code).toBe(1)
32
+ expect(absent.stdout).toBe("")
33
+
34
+ const tail = await invoke("logs", "tail", "--directory", directory)
35
+ expect(tail.code).toBe(0)
36
+ expect(tail.stdout.trim().split("\n").map((line) => JSON.parse(line).event)).toEqual(["process.started", "runtime.status", "log_batch_accepted", "log_frame_committed", "log_closed"])
37
+
38
+ const invalid = await invoke("logs", "search", "--directory", directory, "--client-generation", "not-a-number")
39
+ expect(invalid.code).toBe(2)
40
+ expect(invalid.stderr).toMatch(/Socketduct logs error/)
41
+ }))
42
+
43
+ it("owned CLI follow stays live and emits newly durable writer records", async () => withTestCleanup(async (after) => {
44
+ const root = await mkdtemp(join(tmpdir(), "socketduct-logs-follow-"))
45
+ const directory = join(root, "logs")
46
+ after(() => rm(root, {recursive: true, force: true}))
47
+ const initial = createCompressedJsonlFileLogger({directory, role: "reverse-client"})
48
+ initial.log("process.started", {runtimeMode: "reverse-client"})
49
+ await initial.close()
50
+
51
+ const child = spawn(process.execPath, [cli.pathname, "logs", "tail", "--directory", directory, "--follow", "--poll-ms", "10"], {stdio: ["ignore", "pipe", "pipe"]})
52
+ let stdout = ""
53
+ let stderr = ""
54
+ child.stdout.on("data", (chunk) => { stdout += chunk.toString() })
55
+ child.stderr.on("data", (chunk) => { stderr += chunk.toString() })
56
+ after(() => { if (child.exitCode === null) child.kill("SIGKILL") })
57
+ await waitFor(() => stdout.includes('"eventSequence":"1"'), child, () => stderr)
58
+
59
+ const later = createCompressedJsonlFileLogger({directory, role: "reverse-client"})
60
+ later.log("runtime.status", {virtualStreams: 2})
61
+ await later.close()
62
+ await waitFor(() => stdout.includes('"event":"runtime.status"'), child, () => stderr)
63
+ child.kill("SIGTERM")
64
+ const code = await new Promise((resolve) => child.once("exit", resolve))
65
+ expect(code).toBe(0)
66
+ }))
67
+
68
+ it("owned CLI exits 2 instead of hiding corrupt interior evidence", async () => withTestCleanup(async (after) => {
69
+ const root = await mkdtemp(join(tmpdir(), "socketduct-logs-corrupt-"))
70
+ const directory = join(root, "logs")
71
+ after(() => rm(root, {recursive: true, force: true}))
72
+ const logger = createCompressedJsonlFileLogger({directory, role: "reverse-gateway"})
73
+ logger.log("process.started", {runtimeMode: "reverse-gateway"})
74
+ await logger.close()
75
+ const name = (await readdir(directory)).find((entry) => entry.endsWith(".jsonl.zst"))
76
+ expect(name).toBeDefined()
77
+ const path = join(directory, /** @type {string} */ (name))
78
+ const bytes = await readFile(path)
79
+ bytes[bytes.length - 1] = /** @type {number} */ (bytes[bytes.length - 1]) ^ 0xff
80
+ await writeFile(path, bytes)
81
+
82
+ const result = await invoke("logs", "search", "--directory", directory, "--event", "process.started")
83
+ expect(result.code).toBe(2)
84
+ expect(result.stdout).toBe("")
85
+ expect(result.stderr).toMatch(/Socketduct log evidence error/)
86
+ }))
87
+
88
+ it("tail reader incrementally follows rotation and prunes retention-deleted cursors", async () => withTestCleanup(async (after) => {
89
+ const root = await mkdtemp(join(tmpdir(), "socketduct-logs-incremental-tail-"))
90
+ const directory = join(root, "logs")
91
+ after(() => rm(root, {recursive: true, force: true}))
92
+ const logger = createCompressedJsonlFileLogger({directory, role: "reverse-hub", frameMaxBytes: 256, maxSegmentBytes: 1024, compressionLevel: 1})
93
+ await logger.flush()
94
+ for (let index = 0; index < 8; index += 1) {
95
+ logger.log("process.started", {runtimeMode: "reverse-hub"})
96
+ await logger.flush()
97
+ }
98
+ await logger.flush()
67
99
 
68
- test("owned CLI exits 2 instead of hiding corrupt interior evidence", async (t) => {
69
- const root = await mkdtemp(join(tmpdir(), "socketduct-logs-corrupt-"))
70
- const directory = join(root, "logs")
71
- t.after(() => rm(root, {recursive: true, force: true}))
72
- const logger = createCompressedJsonlFileLogger({directory, role: "reverse-gateway"})
73
- logger.log("process.started", {runtimeMode: "reverse-gateway"})
74
- await logger.close()
75
- const name = (await readdir(directory)).find((entry) => entry.endsWith(".jsonl.zst"))
76
- assert(name !== undefined)
77
- const path = join(directory, name)
78
- const bytes = await readFile(path)
79
- bytes[bytes.length - 1] = /** @type {number} */ (bytes[bytes.length - 1]) ^ 0xff
80
- await writeFile(path, bytes)
81
-
82
- const result = await invoke("logs", "search", "--directory", directory, "--event", "process.started")
83
- assert.equal(result.code, 2)
84
- assert.equal(result.stdout, "")
85
- assert.match(result.stderr, /Socketduct log evidence error/)
86
- })
100
+ const reader = new CompressedLogTailReader(directory)
101
+ const initial = await reader.poll()
102
+ expect(initial.records.length > 0).toBeTruthy()
103
+ const identities = new Set(initial.records.map((record) => `${record.writerEpoch}\0${record.eventSequence}`))
104
+ expect(identities.size).toBe(initial.records.length)
105
+ const decoded = reader.status().decodedFrames
106
+ expect(decoded > 0).toBeTruthy()
107
+ expect((await reader.poll()).records).toEqual([])
108
+ expect(reader.status().decodedFrames).toBe(decoded)
87
109
 
88
- test("tail reader incrementally follows rotation and prunes retention-deleted cursors", async (t) => {
89
- const root = await mkdtemp(join(tmpdir(), "socketduct-logs-incremental-tail-"))
90
- const directory = join(root, "logs")
91
- t.after(() => rm(root, {recursive: true, force: true}))
92
- const logger = createCompressedJsonlFileLogger({directory, role: "reverse-hub", frameMaxBytes: 256, maxSegmentBytes: 1024, compressionLevel: 1})
93
- await logger.flush()
94
- for (let index = 0; index < 8; index += 1) {
95
110
  logger.log("process.started", {runtimeMode: "reverse-hub"})
96
111
  await logger.flush()
97
- }
98
- await logger.flush()
99
-
100
- const reader = new CompressedLogTailReader(directory)
101
- const initial = await reader.poll()
102
- assert(initial.records.length > 0)
103
- const identities = new Set(initial.records.map((record) => `${record.writerEpoch}\0${record.eventSequence}`))
104
- assert.equal(identities.size, initial.records.length)
105
- const decoded = reader.status().decodedFrames
106
- assert(decoded > 0)
107
- assert.deepEqual((await reader.poll()).records, [])
108
- assert.equal(reader.status().decodedFrames, decoded)
109
-
110
- logger.log("process.started", {runtimeMode: "reverse-hub"})
111
- await logger.flush()
112
- await logger.flush()
113
- const activeDelta = await reader.poll()
114
- assert(activeDelta.records.length > 0)
115
- for (const record of activeDelta.records) assert.equal(identities.has(`${record.writerEpoch}\0${record.eventSequence}`), false)
116
- await logger.close()
117
- const closeDelta = await reader.poll()
118
- assert(closeDelta.records.length > 0)
119
- for (const record of closeDelta.records) assert.equal(identities.has(`${record.writerEpoch}\0${record.eventSequence}`), false)
120
-
121
- const oldSegments = (await readdir(directory)).filter((name) => name.endsWith(".jsonl.zst"))
122
- assert(oldSegments.length > 1, "fixture must rotate through more than one segment")
123
- for (const name of oldSegments) await utimes(join(directory, name), new Date(1_000), new Date(1_000))
124
- const later = createCompressedJsonlFileLogger({directory, role: "reverse-hub", compressionLevel: 1})
125
- later.log("runtime.status", {virtualStreams: 2})
126
- await later.close()
127
- const laterSegments = (await readdir(directory)).filter((name) => name.endsWith(".jsonl.zst") && !oldSegments.includes(name))
128
- for (const name of laterSegments) await utimes(join(directory, name), new Date(10_000), new Date(10_000))
129
- const laterDelta = await reader.poll()
130
- assert(laterDelta.records.some((record) => record.event === "runtime.status"))
131
-
132
- await enforceLogRetention({directory, maxAgeMs: 5_000, maxBytes: 1_000_000, now: () => new Date(10_000)})
133
- const beforePruneDecode = reader.status().decodedFrames
134
- assert.deepEqual((await reader.poll()).records, [])
135
- const survivingSegments = (await readdir(directory)).filter((name) => name.endsWith(".jsonl.zst")).length
136
- assert(reader.status().trackedSegments <= survivingSegments)
137
- assert.equal(reader.status().decodedFrames, beforePruneDecode)
112
+ await logger.flush()
113
+ const activeDelta = await reader.poll()
114
+ expect(activeDelta.records.length > 0).toBeTruthy()
115
+ for (const record of activeDelta.records) expect(identities.has(`${record.writerEpoch}\0${record.eventSequence}`)).toBe(false)
116
+ await logger.close()
117
+ const closeDelta = await reader.poll()
118
+ expect(closeDelta.records.length > 0).toBeTruthy()
119
+ for (const record of closeDelta.records) expect(identities.has(`${record.writerEpoch}\0${record.eventSequence}`)).toBe(false)
120
+
121
+ const oldSegments = (await readdir(directory)).filter((name) => name.endsWith(".jsonl.zst"))
122
+ // fixture must rotate through more than one segment
123
+ expect(oldSegments.length > 1).toBeTruthy()
124
+ for (const name of oldSegments) await utimes(join(directory, name), new Date(1_000), new Date(1_000))
125
+ const later = createCompressedJsonlFileLogger({directory, role: "reverse-hub", compressionLevel: 1})
126
+ later.log("runtime.status", {virtualStreams: 2})
127
+ await later.close()
128
+ const laterSegments = (await readdir(directory)).filter((name) => name.endsWith(".jsonl.zst") && !oldSegments.includes(name))
129
+ for (const name of laterSegments) await utimes(join(directory, name), new Date(10_000), new Date(10_000))
130
+ const laterDelta = await reader.poll()
131
+ expect(laterDelta.records.some((record) => record.event === "runtime.status")).toBeTruthy()
132
+
133
+ await enforceLogRetention({directory, maxAgeMs: 5_000, maxBytes: 1_000_000, now: () => new Date(10_000)})
134
+ const beforePruneDecode = reader.status().decodedFrames
135
+ expect((await reader.poll()).records).toEqual([])
136
+ const survivingSegments = (await readdir(directory)).filter((name) => name.endsWith(".jsonl.zst")).length
137
+ expect(reader.status().trackedSegments <= survivingSegments).toBeTruthy()
138
+ expect(reader.status().decodedFrames).toBe(beforePruneDecode)
139
+ }))
138
140
  })
139
141
 
140
142
  /** @param {...string} args - CLI arguments. */
@@ -1,10 +1,9 @@
1
1
  /* eslint-disable jsdoc/escape-inline-tags, jsdoc/require-param-type, jsdoc/require-returns */
2
- import assert from "node:assert/strict"
2
+ import {describe, expect, it, withTestCleanup} from "@velocious/testing"
3
3
  import {createHash, randomBytes} from "node:crypto"
4
4
  import {mkdtemp, readdir, rm, stat, utimes, writeFile} from "node:fs/promises"
5
5
  import {tmpdir} from "node:os"
6
6
  import {join} from "node:path"
7
- import test from "node:test"
8
7
 
9
8
  import {createLogBatchId, inspectLogBatch} from "../../src/logging/log-batch.js"
10
9
  import {readCompressedLogDirectory} from "../../src/logging/compressed-log-reader.js"
@@ -13,108 +12,110 @@ import {enforceLogRetention} from "../../src/logging/log-retention.js"
13
12
  import {createIndexEntry, createIndexHeader, INDEX_ENTRY_BYTES, INDEX_HEADER_BYTES, uuidBytes} from "../../src/logging/log-segment-index.js"
14
13
  import {compressLogFrame} from "../../src/logging/zstd-frame-codec.js"
15
14
 
16
- test("recovery indexes a complete data-synced orphan without duplicating its batch", async (t) => {
17
- const root = await mkdtemp(join(tmpdir(), "socketduct-compressed-orphan-"))
18
- t.after(() => rm(root, {recursive: true, force: true}))
19
- const fixture = await createRecoveryFixture(root, false)
20
- const sink = new CompressedLogWorkerSink(fixture.options)
21
- const recovered = await sink.start()
22
- assert.equal(recovered.durableHighWater, "2")
23
- assert.equal(recovered.orphanRecoveredCount, 1)
24
- assert.equal(recovered.lastDurableBatch?.batchId, fixture.secondBatchId)
25
- assert.equal((await stat(fixture.indexPath)).size, INDEX_HEADER_BYTES + 2 * INDEX_ENTRY_BYTES)
15
+ describe("compressed log recovery", () => {
16
+ it("recovery indexes a complete data-synced orphan without duplicating its batch", async () => withTestCleanup(async (after) => {
17
+ const root = await mkdtemp(join(tmpdir(), "socketduct-compressed-orphan-"))
18
+ after(() => rm(root, {recursive: true, force: true}))
19
+ const fixture = await createRecoveryFixture(root, false)
20
+ const sink = new CompressedLogWorkerSink(fixture.options)
21
+ const recovered = await sink.start()
22
+ expect(recovered.durableHighWater).toBe("2")
23
+ expect(recovered.orphanRecoveredCount).toBe(1)
24
+ expect(recovered.lastDurableBatch?.batchId).toBe(fixture.secondBatchId)
25
+ expect((await stat(fixture.indexPath)).size).toBe(INDEX_HEADER_BYTES + 2 * INDEX_ENTRY_BYTES)
26
26
 
27
- const duplicate = await sink.appendBatch(fixture.secondMessage)
28
- assert.equal(duplicate.duplicate, true)
29
- assert.equal((await stat(fixture.dataPath)).size, fixture.firstFrame.length + fixture.secondFrame.length)
30
- await sink.close()
31
- const result = await readCompressedLogDirectory(root)
32
- assert.equal(result.incomplete, false)
33
- assert.deepEqual(result.records.map((record) => record.eventSequence), ["1", "2"])
34
- })
35
-
36
- test("recovery truncates only a torn final data frame and accepts its replay", async (t) => {
37
- const root = await mkdtemp(join(tmpdir(), "socketduct-compressed-torn-"))
38
- t.after(() => rm(root, {recursive: true, force: true}))
39
- const fixture = await createRecoveryFixture(root, true)
40
- const sink = new CompressedLogWorkerSink(fixture.options)
41
- const recovered = await sink.start()
42
- assert.equal(recovered.durableHighWater, "1")
43
- assert.equal((await stat(fixture.dataPath)).size, fixture.firstFrame.length)
44
-
45
- const replayed = await sink.appendBatch(fixture.secondMessage)
46
- assert.equal(replayed.duplicate, false)
47
- await sink.close()
48
- const result = await readCompressedLogDirectory(root)
49
- assert.equal(result.incomplete, false)
50
- assert.deepEqual(result.records.map((record) => record.eventSequence), ["1", "2"])
51
- })
52
-
53
- test("worker acknowledgements retain only the latest durable batch identity", async (t) => {
54
- const root = await mkdtemp(join(tmpdir(), "socketduct-compressed-bounded-ack-"))
55
- t.after(() => rm(root, {recursive: true, force: true}))
56
- const writerEpoch = randomBytes(16)
57
- const options = {directory: root, role: "reverse-hub", writerEpoch: writerEpoch.toString("base64url"), maxSegmentBytes: 4096, compressionLevel: 19}
58
- const sink = new CompressedLogWorkerSink(options)
59
- await sink.start()
60
- let latest = await sink.appendBatch(batchMessage(writerEpoch, 1))
61
- for (let sequence = 2; sequence <= 40; sequence += 1) latest = await sink.appendBatch(batchMessage(writerEpoch, sequence))
62
- assert.equal(Object.hasOwn(latest, "durableBatchIds"), false)
63
- assert.deepEqual(latest.lastDurableBatch, {
64
- batchId: batchMessage(writerEpoch, 40).batchId,
65
- firstSequence: "40",
66
- lastSequence: "40"
67
- })
68
- assert.equal((await sink.appendBatch(batchMessage(writerEpoch, 40))).duplicate, true)
69
- await sink.close()
27
+ const duplicate = await sink.appendBatch(fixture.secondMessage)
28
+ expect(duplicate.duplicate).toBe(true)
29
+ expect((await stat(fixture.dataPath)).size).toBe(fixture.firstFrame.length + fixture.secondFrame.length)
30
+ await sink.close()
31
+ const result = await readCompressedLogDirectory(root)
32
+ expect(result.incomplete).toBe(false)
33
+ expect(result.records.map((record) => record.eventSequence)).toEqual(["1", "2"])
34
+ }))
70
35
 
71
- const restarted = new CompressedLogWorkerSink(options)
72
- const recovered = await restarted.start()
73
- assert.equal(Object.hasOwn(recovered, "durableBatchIds"), false)
74
- assert.deepEqual(recovered.lastDurableBatch, latest.lastDurableBatch)
75
- await restarted.close()
76
- })
36
+ it("recovery truncates only a torn final data frame and accepts its replay", async () => withTestCleanup(async (after) => {
37
+ const root = await mkdtemp(join(tmpdir(), "socketduct-compressed-torn-"))
38
+ after(() => rm(root, {recursive: true, force: true}))
39
+ const fixture = await createRecoveryFixture(root, true)
40
+ const sink = new CompressedLogWorkerSink(fixture.options)
41
+ const recovered = await sink.start()
42
+ expect(recovered.durableHighWater).toBe("1")
43
+ expect((await stat(fixture.dataPath)).size).toBe(fixture.firstFrame.length)
77
44
 
78
- test("recovery accepts a retention-deleted prefix but rejects interior gaps", async (t) => {
79
- const root = await mkdtemp(join(tmpdir(), "socketduct-compressed-retained-prefix-"))
80
- t.after(() => rm(root, {recursive: true, force: true}))
81
- const writerEpoch = randomBytes(16)
82
- const options = {directory: root, role: "reverse-hub", writerEpoch: writerEpoch.toString("base64url"), maxSegmentBytes: 4096, compressionLevel: 19, retentionBytes: 1_000_000}
45
+ const replayed = await sink.appendBatch(fixture.secondMessage)
46
+ expect(replayed.duplicate).toBe(false)
47
+ await sink.close()
48
+ const result = await readCompressedLogDirectory(root)
49
+ expect(result.incomplete).toBe(false)
50
+ expect(result.records.map((record) => record.eventSequence)).toEqual(["1", "2"])
51
+ }))
83
52
 
84
- const segments = []
85
- for (let sequence = 1; sequence <= 3; sequence += 1) {
53
+ it("worker acknowledgements retain only the latest durable batch identity", async () => withTestCleanup(async (after) => {
54
+ const root = await mkdtemp(join(tmpdir(), "socketduct-compressed-bounded-ack-"))
55
+ after(() => rm(root, {recursive: true, force: true}))
56
+ const writerEpoch = randomBytes(16)
57
+ const options = {directory: root, role: "reverse-hub", writerEpoch: writerEpoch.toString("base64url"), maxSegmentBytes: 4096, compressionLevel: 19}
86
58
  const sink = new CompressedLogWorkerSink(options)
87
59
  await sink.start()
88
- await sink.appendBatch(batchMessage(writerEpoch, sequence))
89
- segments.push(/** @type {string} */ (sink.status().segmentPath))
60
+ let latest = await sink.appendBatch(batchMessage(writerEpoch, 1))
61
+ for (let sequence = 2; sequence <= 40; sequence += 1) latest = await sink.appendBatch(batchMessage(writerEpoch, sequence))
62
+ expect(Object.hasOwn(latest, "durableBatchIds")).toBe(false)
63
+ expect(latest.lastDurableBatch).toEqual({
64
+ batchId: batchMessage(writerEpoch, 40).batchId,
65
+ firstSequence: "40",
66
+ lastSequence: "40"
67
+ })
68
+ expect((await sink.appendBatch(batchMessage(writerEpoch, 40))).duplicate).toBe(true)
90
69
  await sink.close()
91
- }
92
- assert.equal(segments.length, 3)
93
- for (let index = 0; index < segments.length; index += 1) {
94
- await utimes(/** @type {string} */ (segments[index]), new Date(1_000 + index), new Date(1_000 + index))
95
- }
96
- await enforceLogRetention({directory: root, maxAgeMs: 1, maxBytes: 1_000_000, now: () => new Date(1_003)})
97
- assert.equal((await readdir(root)).filter((name) => name.endsWith(".jsonl.zst")).length, 1)
98
70
 
99
- const recovered = new CompressedLogWorkerSink(options)
100
- assert.equal((await recovered.start()).durableHighWater, "3")
101
- assert.equal((await recovered.appendBatch(batchMessage(writerEpoch, 4))).durableHighWater, "4")
102
- await recovered.close()
71
+ const restarted = new CompressedLogWorkerSink(options)
72
+ const recovered = await restarted.start()
73
+ expect(Object.hasOwn(recovered, "durableBatchIds")).toBe(false)
74
+ expect(recovered.lastDurableBatch).toEqual(latest.lastDurableBatch)
75
+ await restarted.close()
76
+ }))
103
77
 
104
- const gapDirectory = join(root, "interior-gap")
105
- const gapEpoch = randomBytes(16)
106
- const gapOptions = {...options, directory: gapDirectory, writerEpoch: gapEpoch.toString("base64url")}
107
- const gapSegments = []
108
- for (let sequence = 1; sequence <= 3; sequence += 1) {
109
- const sink = new CompressedLogWorkerSink(gapOptions)
110
- await sink.start()
111
- await sink.appendBatch(batchMessage(gapEpoch, sequence))
112
- gapSegments.push(/** @type {string} */ (sink.status().segmentPath))
113
- await sink.close()
114
- }
115
- const missing = /** @type {string} */ (gapSegments[1])
116
- await Promise.all([rm(missing), rm(`${missing}.idx`), rm(`${missing}.sealed`)])
117
- await assert.rejects(() => new CompressedLogWorkerSink(gapOptions).start(), /interior sequence gap/)
78
+ it("recovery accepts a retention-deleted prefix but rejects interior gaps", async () => withTestCleanup(async (after) => {
79
+ const root = await mkdtemp(join(tmpdir(), "socketduct-compressed-retained-prefix-"))
80
+ after(() => rm(root, {recursive: true, force: true}))
81
+ const writerEpoch = randomBytes(16)
82
+ const options = {directory: root, role: "reverse-hub", writerEpoch: writerEpoch.toString("base64url"), maxSegmentBytes: 4096, compressionLevel: 19, retentionBytes: 1_000_000}
83
+
84
+ const segments = []
85
+ for (let sequence = 1; sequence <= 3; sequence += 1) {
86
+ const sink = new CompressedLogWorkerSink(options)
87
+ await sink.start()
88
+ await sink.appendBatch(batchMessage(writerEpoch, sequence))
89
+ segments.push(/** @type {string} */ (sink.status().segmentPath))
90
+ await sink.close()
91
+ }
92
+ expect(segments.length).toBe(3)
93
+ for (let index = 0; index < segments.length; index += 1) {
94
+ await utimes(/** @type {string} */ (segments[index]), new Date(1_000 + index), new Date(1_000 + index))
95
+ }
96
+ await enforceLogRetention({directory: root, maxAgeMs: 1, maxBytes: 1_000_000, now: () => new Date(1_003)})
97
+ expect((await readdir(root)).filter((name) => name.endsWith(".jsonl.zst")).length).toBe(1)
98
+
99
+ const recovered = new CompressedLogWorkerSink(options)
100
+ expect((await recovered.start()).durableHighWater).toBe("3")
101
+ expect((await recovered.appendBatch(batchMessage(writerEpoch, 4))).durableHighWater).toBe("4")
102
+ await recovered.close()
103
+
104
+ const gapDirectory = join(root, "interior-gap")
105
+ const gapEpoch = randomBytes(16)
106
+ const gapOptions = {...options, directory: gapDirectory, writerEpoch: gapEpoch.toString("base64url")}
107
+ const gapSegments = []
108
+ for (let sequence = 1; sequence <= 3; sequence += 1) {
109
+ const sink = new CompressedLogWorkerSink(gapOptions)
110
+ await sink.start()
111
+ await sink.appendBatch(batchMessage(gapEpoch, sequence))
112
+ gapSegments.push(/** @type {string} */ (sink.status().segmentPath))
113
+ await sink.close()
114
+ }
115
+ const missing = /** @type {string} */ (gapSegments[1])
116
+ await Promise.all([rm(missing), rm(`${missing}.idx`), rm(`${missing}.sealed`)])
117
+ await expect(new CompressedLogWorkerSink(gapOptions).start()).rejects.toThrow(/interior sequence gap/)
118
+ }))
118
119
  })
119
120
 
120
121
  /**