socketduct 0.1.25 → 0.1.27

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/AGENTS.md +2 -2
  2. package/changelog.d/20261010130509-bound-log-frame-compression-memory.md +1 -0
  3. package/changelog.d/20261025070000-reclaim-unreadable-retention-lock.md +14 -0
  4. package/docs/runbooks/jsonl-integrity-rollout.md +8 -6
  5. package/package.json +2 -2
  6. package/scripts/test.js +9 -14
  7. package/src/logging/compressed-log-writer.js +1 -1
  8. package/src/logging/log-retention.js +36 -1
  9. package/src/logging/zstd-frame-codec.js +15 -1
  10. package/test/e2e/gateway-process.test.js +93 -92
  11. package/test/e2e/logger-shutdown-process.test.js +40 -37
  12. package/test/e2e/reverse-client-recovery-process.test.js +107 -103
  13. package/test/e2e/reverse-gateway-process.test.js +328 -318
  14. package/test/integration/initial-target-recovery.test.js +333 -334
  15. package/test/integration/logging-cli.test.js +121 -119
  16. package/test/integration/logging-compressed-recovery.test.js +96 -95
  17. package/test/integration/multiplexed-compatibility.test.js +178 -177
  18. package/test/integration/multiplexed-corrections.test.js +360 -359
  19. package/test/integration/multiplexed-failures.test.js +253 -251
  20. package/test/integration/multiplexed-transport-set.test.js +334 -322
  21. package/test/integration/multiplexed-transport.test.js +135 -134
  22. package/test/integration/reverse-gateway-stream-id-reset.test.js +90 -89
  23. package/test/integration/reverse-gateway.test.js +2309 -2218
  24. package/test/integration/reverse-resume-continuity.test.js +219 -218
  25. package/test/integration/reverse-route-class.test.js +358 -341
  26. package/test/integration/socketduct-http-agent.test.js +314 -312
  27. package/test/integration/socketduct-https-agent.test.js +516 -485
  28. package/test/integration/stream-compression.test.js +862 -824
  29. package/test/integration/tcp-forwarding.test.js +2550 -2447
  30. package/test/integration/tls-forwarding.test.js +593 -592
  31. package/test/unit/auth/auth.test.js +43 -44
  32. package/test/unit/auth/static-token-store.test.js +9 -8
  33. package/test/unit/client/connect.test.js +41 -40
  34. package/test/unit/client/deferred-virtual-socket.test.js +95 -94
  35. package/test/unit/client/multiplexed-spool-budget.test.js +155 -154
  36. package/test/unit/client/multiplexed-transport-set.test.js +830 -748
  37. package/test/unit/client/socketduct-socket.test.js +768 -771
  38. package/test/unit/client/virtual-socket.test.js +198 -168
  39. package/test/unit/config/docker-gateway-files.test.js +224 -224
  40. package/test/unit/config/environment.test.js +472 -492
  41. package/test/unit/index.test.js +53 -52
  42. package/test/unit/logging/compressed-log-worker.test.js +23 -22
  43. package/test/unit/logging/compressed-log-writer.test.js +155 -154
  44. package/test/unit/logging/log-retention.test.js +71 -48
  45. package/test/unit/logging/log-segment-index.test.js +84 -81
  46. package/test/unit/logging/zstd-frame-codec.test.js +28 -19
  47. package/test/unit/protocol/compression.test.js +254 -298
  48. package/test/unit/protocol/frame.test.js +280 -276
  49. package/test/unit/protocol/v3-compression.test.js +21 -23
  50. package/test/unit/protocol/v3-frame.test.js +42 -41
  51. package/test/unit/release-configuration.test.js +15 -12
  52. package/test/unit/reverse/disconnect-attribution.test.js +65 -64
  53. package/test/unit/reverse-terminal-outcomes.test.js +53 -50
  54. package/test/unit/server/control-handler.test.js +232 -231
  55. package/test/unit/server/primary.test.js +51 -49
  56. package/test/unit/server/relay-runtime.test.js +280 -294
  57. package/test/unit/sessions/session.test.js +48 -62
  58. package/test/unit/spool/spool-file.test.js +401 -385
@@ -1,10 +1,9 @@
1
1
  /* eslint-disable jsdoc/escape-inline-tags, jsdoc/require-param, jsdoc/require-returns */
2
- import assert from "node:assert/strict"
2
+ import {describe, expect, it, withTestCleanup} from "@velocious/testing"
3
3
  import {execFile, spawn} from "node:child_process"
4
4
  import {mkdtemp, readFile, readdir, rm, utimes, writeFile} from "node:fs/promises"
5
5
  import {tmpdir} from "node:os"
6
6
  import {join} from "node:path"
7
- import test from "node:test"
8
7
  import {promisify} from "node:util"
9
8
 
10
9
  import {createCompressedJsonlFileLogger} from "../../src/logging/compressed-log-writer.js"
@@ -14,127 +13,130 @@ import {enforceLogRetention} from "../../src/logging/log-retention.js"
14
13
  const run = promisify(execFile)
15
14
  const cli = new URL("../../src/cli/socketduct.js", import.meta.url)
16
15
 
17
- test("owned CLI searches and tails durable compressed records with stable exits", async (t) => {
18
- const root = await mkdtemp(join(tmpdir(), "socketduct-logs-cli-"))
19
- const directory = join(root, "logs")
20
- t.after(() => rm(root, {recursive: true, force: true}))
21
- const logger = createCompressedJsonlFileLogger({directory, role: "reverse-hub"})
22
- logger.log("process.started", {runtimeMode: "reverse-hub"})
23
- logger.log("runtime.status", {virtualStreams: 7})
24
- await logger.close()
25
-
26
- const match = await invoke("logs", "search", "--directory", directory, "--event", "runtime.status")
27
- assert.equal(match.code, 0)
28
- assert.deepEqual(match.stdout.trim().split("\n").map((line) => JSON.parse(line).event), ["runtime.status"])
29
-
30
- const absent = await invoke("logs", "search", "--directory", directory, "--event", "transport.failed")
31
- assert.equal(absent.code, 1)
32
- assert.equal(absent.stdout, "")
33
-
34
- const tail = await invoke("logs", "tail", "--directory", directory)
35
- assert.equal(tail.code, 0)
36
- assert.deepEqual(tail.stdout.trim().split("\n").map((line) => JSON.parse(line).event), ["process.started", "runtime.status", "log_batch_accepted", "log_frame_committed", "log_closed"])
37
-
38
- const invalid = await invoke("logs", "search", "--directory", directory, "--client-generation", "not-a-number")
39
- assert.equal(invalid.code, 2)
40
- assert.match(invalid.stderr, /Socketduct logs error/)
41
- })
42
-
43
- test("owned CLI follow stays live and emits newly durable writer records", async (t) => {
44
- const root = await mkdtemp(join(tmpdir(), "socketduct-logs-follow-"))
45
- const directory = join(root, "logs")
46
- t.after(() => rm(root, {recursive: true, force: true}))
47
- const initial = createCompressedJsonlFileLogger({directory, role: "reverse-client"})
48
- initial.log("process.started", {runtimeMode: "reverse-client"})
49
- await initial.close()
50
-
51
- const child = spawn(process.execPath, [cli.pathname, "logs", "tail", "--directory", directory, "--follow", "--poll-ms", "10"], {stdio: ["ignore", "pipe", "pipe"]})
52
- let stdout = ""
53
- let stderr = ""
54
- child.stdout.on("data", (chunk) => { stdout += chunk.toString() })
55
- child.stderr.on("data", (chunk) => { stderr += chunk.toString() })
56
- t.after(() => { if (child.exitCode === null) child.kill("SIGKILL") })
57
- await waitFor(() => stdout.includes('"eventSequence":"1"'), child, () => stderr)
58
-
59
- const later = createCompressedJsonlFileLogger({directory, role: "reverse-client"})
60
- later.log("runtime.status", {virtualStreams: 2})
61
- await later.close()
62
- await waitFor(() => stdout.includes('"event":"runtime.status"'), child, () => stderr)
63
- child.kill("SIGTERM")
64
- const code = await new Promise((resolve) => child.once("exit", resolve))
65
- assert.equal(code, 0)
66
- })
16
+ describe("socketduct logs CLI", () => {
17
+ it("owned CLI searches and tails durable compressed records with stable exits", async () => withTestCleanup(async (after) => {
18
+ const root = await mkdtemp(join(tmpdir(), "socketduct-logs-cli-"))
19
+ const directory = join(root, "logs")
20
+ after(() => rm(root, {recursive: true, force: true}))
21
+ const logger = createCompressedJsonlFileLogger({directory, role: "reverse-hub"})
22
+ logger.log("process.started", {runtimeMode: "reverse-hub"})
23
+ logger.log("runtime.status", {virtualStreams: 7})
24
+ await logger.close()
25
+
26
+ const match = await invoke("logs", "search", "--directory", directory, "--event", "runtime.status")
27
+ expect(match.code).toBe(0)
28
+ expect(match.stdout.trim().split("\n").map((line) => JSON.parse(line).event)).toEqual(["runtime.status"])
29
+
30
+ const absent = await invoke("logs", "search", "--directory", directory, "--event", "transport.failed")
31
+ expect(absent.code).toBe(1)
32
+ expect(absent.stdout).toBe("")
33
+
34
+ const tail = await invoke("logs", "tail", "--directory", directory)
35
+ expect(tail.code).toBe(0)
36
+ expect(tail.stdout.trim().split("\n").map((line) => JSON.parse(line).event)).toEqual(["process.started", "runtime.status", "log_batch_accepted", "log_frame_committed", "log_closed"])
37
+
38
+ const invalid = await invoke("logs", "search", "--directory", directory, "--client-generation", "not-a-number")
39
+ expect(invalid.code).toBe(2)
40
+ expect(invalid.stderr).toMatch(/Socketduct logs error/)
41
+ }))
42
+
43
+ it("owned CLI follow stays live and emits newly durable writer records", async () => withTestCleanup(async (after) => {
44
+ const root = await mkdtemp(join(tmpdir(), "socketduct-logs-follow-"))
45
+ const directory = join(root, "logs")
46
+ after(() => rm(root, {recursive: true, force: true}))
47
+ const initial = createCompressedJsonlFileLogger({directory, role: "reverse-client"})
48
+ initial.log("process.started", {runtimeMode: "reverse-client"})
49
+ await initial.close()
50
+
51
+ const child = spawn(process.execPath, [cli.pathname, "logs", "tail", "--directory", directory, "--follow", "--poll-ms", "10"], {stdio: ["ignore", "pipe", "pipe"]})
52
+ let stdout = ""
53
+ let stderr = ""
54
+ child.stdout.on("data", (chunk) => { stdout += chunk.toString() })
55
+ child.stderr.on("data", (chunk) => { stderr += chunk.toString() })
56
+ after(() => { if (child.exitCode === null) child.kill("SIGKILL") })
57
+ await waitFor(() => stdout.includes('"eventSequence":"1"'), child, () => stderr)
58
+
59
+ const later = createCompressedJsonlFileLogger({directory, role: "reverse-client"})
60
+ later.log("runtime.status", {virtualStreams: 2})
61
+ await later.close()
62
+ await waitFor(() => stdout.includes('"event":"runtime.status"'), child, () => stderr)
63
+ child.kill("SIGTERM")
64
+ const code = await new Promise((resolve) => child.once("exit", resolve))
65
+ expect(code).toBe(0)
66
+ }))
67
+
68
+ it("owned CLI exits 2 instead of hiding corrupt interior evidence", async () => withTestCleanup(async (after) => {
69
+ const root = await mkdtemp(join(tmpdir(), "socketduct-logs-corrupt-"))
70
+ const directory = join(root, "logs")
71
+ after(() => rm(root, {recursive: true, force: true}))
72
+ const logger = createCompressedJsonlFileLogger({directory, role: "reverse-gateway"})
73
+ logger.log("process.started", {runtimeMode: "reverse-gateway"})
74
+ await logger.close()
75
+ const name = (await readdir(directory)).find((entry) => entry.endsWith(".jsonl.zst"))
76
+ expect(name).toBeDefined()
77
+ const path = join(directory, /** @type {string} */ (name))
78
+ const bytes = await readFile(path)
79
+ bytes[bytes.length - 1] = /** @type {number} */ (bytes[bytes.length - 1]) ^ 0xff
80
+ await writeFile(path, bytes)
81
+
82
+ const result = await invoke("logs", "search", "--directory", directory, "--event", "process.started")
83
+ expect(result.code).toBe(2)
84
+ expect(result.stdout).toBe("")
85
+ expect(result.stderr).toMatch(/Socketduct log evidence error/)
86
+ }))
87
+
88
+ it("tail reader incrementally follows rotation and prunes retention-deleted cursors", async () => withTestCleanup(async (after) => {
89
+ const root = await mkdtemp(join(tmpdir(), "socketduct-logs-incremental-tail-"))
90
+ const directory = join(root, "logs")
91
+ after(() => rm(root, {recursive: true, force: true}))
92
+ const logger = createCompressedJsonlFileLogger({directory, role: "reverse-hub", frameMaxBytes: 256, maxSegmentBytes: 1024, compressionLevel: 1})
93
+ await logger.flush()
94
+ for (let index = 0; index < 8; index += 1) {
95
+ logger.log("process.started", {runtimeMode: "reverse-hub"})
96
+ await logger.flush()
97
+ }
98
+ await logger.flush()
67
99
 
68
- test("owned CLI exits 2 instead of hiding corrupt interior evidence", async (t) => {
69
- const root = await mkdtemp(join(tmpdir(), "socketduct-logs-corrupt-"))
70
- const directory = join(root, "logs")
71
- t.after(() => rm(root, {recursive: true, force: true}))
72
- const logger = createCompressedJsonlFileLogger({directory, role: "reverse-gateway"})
73
- logger.log("process.started", {runtimeMode: "reverse-gateway"})
74
- await logger.close()
75
- const name = (await readdir(directory)).find((entry) => entry.endsWith(".jsonl.zst"))
76
- assert(name !== undefined)
77
- const path = join(directory, name)
78
- const bytes = await readFile(path)
79
- bytes[bytes.length - 1] = /** @type {number} */ (bytes[bytes.length - 1]) ^ 0xff
80
- await writeFile(path, bytes)
81
-
82
- const result = await invoke("logs", "search", "--directory", directory, "--event", "process.started")
83
- assert.equal(result.code, 2)
84
- assert.equal(result.stdout, "")
85
- assert.match(result.stderr, /Socketduct log evidence error/)
86
- })
100
+ const reader = new CompressedLogTailReader(directory)
101
+ const initial = await reader.poll()
102
+ expect(initial.records.length > 0).toBeTruthy()
103
+ const identities = new Set(initial.records.map((record) => `${record.writerEpoch}\0${record.eventSequence}`))
104
+ expect(identities.size).toBe(initial.records.length)
105
+ const decoded = reader.status().decodedFrames
106
+ expect(decoded > 0).toBeTruthy()
107
+ expect((await reader.poll()).records).toEqual([])
108
+ expect(reader.status().decodedFrames).toBe(decoded)
87
109
 
88
- test("tail reader incrementally follows rotation and prunes retention-deleted cursors", async (t) => {
89
- const root = await mkdtemp(join(tmpdir(), "socketduct-logs-incremental-tail-"))
90
- const directory = join(root, "logs")
91
- t.after(() => rm(root, {recursive: true, force: true}))
92
- const logger = createCompressedJsonlFileLogger({directory, role: "reverse-hub", frameMaxBytes: 256, maxSegmentBytes: 1024, compressionLevel: 1})
93
- await logger.flush()
94
- for (let index = 0; index < 8; index += 1) {
95
110
  logger.log("process.started", {runtimeMode: "reverse-hub"})
96
111
  await logger.flush()
97
- }
98
- await logger.flush()
99
-
100
- const reader = new CompressedLogTailReader(directory)
101
- const initial = await reader.poll()
102
- assert(initial.records.length > 0)
103
- const identities = new Set(initial.records.map((record) => `${record.writerEpoch}\0${record.eventSequence}`))
104
- assert.equal(identities.size, initial.records.length)
105
- const decoded = reader.status().decodedFrames
106
- assert(decoded > 0)
107
- assert.deepEqual((await reader.poll()).records, [])
108
- assert.equal(reader.status().decodedFrames, decoded)
109
-
110
- logger.log("process.started", {runtimeMode: "reverse-hub"})
111
- await logger.flush()
112
- await logger.flush()
113
- const activeDelta = await reader.poll()
114
- assert(activeDelta.records.length > 0)
115
- for (const record of activeDelta.records) assert.equal(identities.has(`${record.writerEpoch}\0${record.eventSequence}`), false)
116
- await logger.close()
117
- const closeDelta = await reader.poll()
118
- assert(closeDelta.records.length > 0)
119
- for (const record of closeDelta.records) assert.equal(identities.has(`${record.writerEpoch}\0${record.eventSequence}`), false)
120
-
121
- const oldSegments = (await readdir(directory)).filter((name) => name.endsWith(".jsonl.zst"))
122
- assert(oldSegments.length > 1, "fixture must rotate through more than one segment")
123
- for (const name of oldSegments) await utimes(join(directory, name), new Date(1_000), new Date(1_000))
124
- const later = createCompressedJsonlFileLogger({directory, role: "reverse-hub", compressionLevel: 1})
125
- later.log("runtime.status", {virtualStreams: 2})
126
- await later.close()
127
- const laterSegments = (await readdir(directory)).filter((name) => name.endsWith(".jsonl.zst") && !oldSegments.includes(name))
128
- for (const name of laterSegments) await utimes(join(directory, name), new Date(10_000), new Date(10_000))
129
- const laterDelta = await reader.poll()
130
- assert(laterDelta.records.some((record) => record.event === "runtime.status"))
131
-
132
- await enforceLogRetention({directory, maxAgeMs: 5_000, maxBytes: 1_000_000, now: () => new Date(10_000)})
133
- const beforePruneDecode = reader.status().decodedFrames
134
- assert.deepEqual((await reader.poll()).records, [])
135
- const survivingSegments = (await readdir(directory)).filter((name) => name.endsWith(".jsonl.zst")).length
136
- assert(reader.status().trackedSegments <= survivingSegments)
137
- assert.equal(reader.status().decodedFrames, beforePruneDecode)
112
+ await logger.flush()
113
+ const activeDelta = await reader.poll()
114
+ expect(activeDelta.records.length > 0).toBeTruthy()
115
+ for (const record of activeDelta.records) expect(identities.has(`${record.writerEpoch}\0${record.eventSequence}`)).toBe(false)
116
+ await logger.close()
117
+ const closeDelta = await reader.poll()
118
+ expect(closeDelta.records.length > 0).toBeTruthy()
119
+ for (const record of closeDelta.records) expect(identities.has(`${record.writerEpoch}\0${record.eventSequence}`)).toBe(false)
120
+
121
+ const oldSegments = (await readdir(directory)).filter((name) => name.endsWith(".jsonl.zst"))
122
+ // fixture must rotate through more than one segment
123
+ expect(oldSegments.length > 1).toBeTruthy()
124
+ for (const name of oldSegments) await utimes(join(directory, name), new Date(1_000), new Date(1_000))
125
+ const later = createCompressedJsonlFileLogger({directory, role: "reverse-hub", compressionLevel: 1})
126
+ later.log("runtime.status", {virtualStreams: 2})
127
+ await later.close()
128
+ const laterSegments = (await readdir(directory)).filter((name) => name.endsWith(".jsonl.zst") && !oldSegments.includes(name))
129
+ for (const name of laterSegments) await utimes(join(directory, name), new Date(10_000), new Date(10_000))
130
+ const laterDelta = await reader.poll()
131
+ expect(laterDelta.records.some((record) => record.event === "runtime.status")).toBeTruthy()
132
+
133
+ await enforceLogRetention({directory, maxAgeMs: 5_000, maxBytes: 1_000_000, now: () => new Date(10_000)})
134
+ const beforePruneDecode = reader.status().decodedFrames
135
+ expect((await reader.poll()).records).toEqual([])
136
+ const survivingSegments = (await readdir(directory)).filter((name) => name.endsWith(".jsonl.zst")).length
137
+ expect(reader.status().trackedSegments <= survivingSegments).toBeTruthy()
138
+ expect(reader.status().decodedFrames).toBe(beforePruneDecode)
139
+ }))
138
140
  })
139
141
 
140
142
  /** @param {...string} args - CLI arguments. */
@@ -1,10 +1,9 @@
1
1
  /* eslint-disable jsdoc/escape-inline-tags, jsdoc/require-param-type, jsdoc/require-returns */
2
- import assert from "node:assert/strict"
2
+ import {describe, expect, it, withTestCleanup} from "@velocious/testing"
3
3
  import {createHash, randomBytes} from "node:crypto"
4
4
  import {mkdtemp, readdir, rm, stat, utimes, writeFile} from "node:fs/promises"
5
5
  import {tmpdir} from "node:os"
6
6
  import {join} from "node:path"
7
- import test from "node:test"
8
7
 
9
8
  import {createLogBatchId, inspectLogBatch} from "../../src/logging/log-batch.js"
10
9
  import {readCompressedLogDirectory} from "../../src/logging/compressed-log-reader.js"
@@ -13,108 +12,110 @@ import {enforceLogRetention} from "../../src/logging/log-retention.js"
13
12
  import {createIndexEntry, createIndexHeader, INDEX_ENTRY_BYTES, INDEX_HEADER_BYTES, uuidBytes} from "../../src/logging/log-segment-index.js"
14
13
  import {compressLogFrame} from "../../src/logging/zstd-frame-codec.js"
15
14
 
16
- test("recovery indexes a complete data-synced orphan without duplicating its batch", async (t) => {
17
- const root = await mkdtemp(join(tmpdir(), "socketduct-compressed-orphan-"))
18
- t.after(() => rm(root, {recursive: true, force: true}))
19
- const fixture = await createRecoveryFixture(root, false)
20
- const sink = new CompressedLogWorkerSink(fixture.options)
21
- const recovered = await sink.start()
22
- assert.equal(recovered.durableHighWater, "2")
23
- assert.equal(recovered.orphanRecoveredCount, 1)
24
- assert.equal(recovered.lastDurableBatch?.batchId, fixture.secondBatchId)
25
- assert.equal((await stat(fixture.indexPath)).size, INDEX_HEADER_BYTES + 2 * INDEX_ENTRY_BYTES)
15
+ describe("compressed log recovery", () => {
16
+ it("recovery indexes a complete data-synced orphan without duplicating its batch", async () => withTestCleanup(async (after) => {
17
+ const root = await mkdtemp(join(tmpdir(), "socketduct-compressed-orphan-"))
18
+ after(() => rm(root, {recursive: true, force: true}))
19
+ const fixture = await createRecoveryFixture(root, false)
20
+ const sink = new CompressedLogWorkerSink(fixture.options)
21
+ const recovered = await sink.start()
22
+ expect(recovered.durableHighWater).toBe("2")
23
+ expect(recovered.orphanRecoveredCount).toBe(1)
24
+ expect(recovered.lastDurableBatch?.batchId).toBe(fixture.secondBatchId)
25
+ expect((await stat(fixture.indexPath)).size).toBe(INDEX_HEADER_BYTES + 2 * INDEX_ENTRY_BYTES)
26
26
 
27
- const duplicate = await sink.appendBatch(fixture.secondMessage)
28
- assert.equal(duplicate.duplicate, true)
29
- assert.equal((await stat(fixture.dataPath)).size, fixture.firstFrame.length + fixture.secondFrame.length)
30
- await sink.close()
31
- const result = await readCompressedLogDirectory(root)
32
- assert.equal(result.incomplete, false)
33
- assert.deepEqual(result.records.map((record) => record.eventSequence), ["1", "2"])
34
- })
35
-
36
- test("recovery truncates only a torn final data frame and accepts its replay", async (t) => {
37
- const root = await mkdtemp(join(tmpdir(), "socketduct-compressed-torn-"))
38
- t.after(() => rm(root, {recursive: true, force: true}))
39
- const fixture = await createRecoveryFixture(root, true)
40
- const sink = new CompressedLogWorkerSink(fixture.options)
41
- const recovered = await sink.start()
42
- assert.equal(recovered.durableHighWater, "1")
43
- assert.equal((await stat(fixture.dataPath)).size, fixture.firstFrame.length)
44
-
45
- const replayed = await sink.appendBatch(fixture.secondMessage)
46
- assert.equal(replayed.duplicate, false)
47
- await sink.close()
48
- const result = await readCompressedLogDirectory(root)
49
- assert.equal(result.incomplete, false)
50
- assert.deepEqual(result.records.map((record) => record.eventSequence), ["1", "2"])
51
- })
52
-
53
- test("worker acknowledgements retain only the latest durable batch identity", async (t) => {
54
- const root = await mkdtemp(join(tmpdir(), "socketduct-compressed-bounded-ack-"))
55
- t.after(() => rm(root, {recursive: true, force: true}))
56
- const writerEpoch = randomBytes(16)
57
- const options = {directory: root, role: "reverse-hub", writerEpoch: writerEpoch.toString("base64url"), maxSegmentBytes: 4096, compressionLevel: 19}
58
- const sink = new CompressedLogWorkerSink(options)
59
- await sink.start()
60
- let latest = await sink.appendBatch(batchMessage(writerEpoch, 1))
61
- for (let sequence = 2; sequence <= 40; sequence += 1) latest = await sink.appendBatch(batchMessage(writerEpoch, sequence))
62
- assert.equal(Object.hasOwn(latest, "durableBatchIds"), false)
63
- assert.deepEqual(latest.lastDurableBatch, {
64
- batchId: batchMessage(writerEpoch, 40).batchId,
65
- firstSequence: "40",
66
- lastSequence: "40"
67
- })
68
- assert.equal((await sink.appendBatch(batchMessage(writerEpoch, 40))).duplicate, true)
69
- await sink.close()
27
+ const duplicate = await sink.appendBatch(fixture.secondMessage)
28
+ expect(duplicate.duplicate).toBe(true)
29
+ expect((await stat(fixture.dataPath)).size).toBe(fixture.firstFrame.length + fixture.secondFrame.length)
30
+ await sink.close()
31
+ const result = await readCompressedLogDirectory(root)
32
+ expect(result.incomplete).toBe(false)
33
+ expect(result.records.map((record) => record.eventSequence)).toEqual(["1", "2"])
34
+ }))
70
35
 
71
- const restarted = new CompressedLogWorkerSink(options)
72
- const recovered = await restarted.start()
73
- assert.equal(Object.hasOwn(recovered, "durableBatchIds"), false)
74
- assert.deepEqual(recovered.lastDurableBatch, latest.lastDurableBatch)
75
- await restarted.close()
76
- })
36
+ it("recovery truncates only a torn final data frame and accepts its replay", async () => withTestCleanup(async (after) => {
37
+ const root = await mkdtemp(join(tmpdir(), "socketduct-compressed-torn-"))
38
+ after(() => rm(root, {recursive: true, force: true}))
39
+ const fixture = await createRecoveryFixture(root, true)
40
+ const sink = new CompressedLogWorkerSink(fixture.options)
41
+ const recovered = await sink.start()
42
+ expect(recovered.durableHighWater).toBe("1")
43
+ expect((await stat(fixture.dataPath)).size).toBe(fixture.firstFrame.length)
77
44
 
78
- test("recovery accepts a retention-deleted prefix but rejects interior gaps", async (t) => {
79
- const root = await mkdtemp(join(tmpdir(), "socketduct-compressed-retained-prefix-"))
80
- t.after(() => rm(root, {recursive: true, force: true}))
81
- const writerEpoch = randomBytes(16)
82
- const options = {directory: root, role: "reverse-hub", writerEpoch: writerEpoch.toString("base64url"), maxSegmentBytes: 4096, compressionLevel: 19, retentionBytes: 1_000_000}
45
+ const replayed = await sink.appendBatch(fixture.secondMessage)
46
+ expect(replayed.duplicate).toBe(false)
47
+ await sink.close()
48
+ const result = await readCompressedLogDirectory(root)
49
+ expect(result.incomplete).toBe(false)
50
+ expect(result.records.map((record) => record.eventSequence)).toEqual(["1", "2"])
51
+ }))
83
52
 
84
- const segments = []
85
- for (let sequence = 1; sequence <= 3; sequence += 1) {
53
+ it("worker acknowledgements retain only the latest durable batch identity", async () => withTestCleanup(async (after) => {
54
+ const root = await mkdtemp(join(tmpdir(), "socketduct-compressed-bounded-ack-"))
55
+ after(() => rm(root, {recursive: true, force: true}))
56
+ const writerEpoch = randomBytes(16)
57
+ const options = {directory: root, role: "reverse-hub", writerEpoch: writerEpoch.toString("base64url"), maxSegmentBytes: 4096, compressionLevel: 19}
86
58
  const sink = new CompressedLogWorkerSink(options)
87
59
  await sink.start()
88
- await sink.appendBatch(batchMessage(writerEpoch, sequence))
89
- segments.push(/** @type {string} */ (sink.status().segmentPath))
60
+ let latest = await sink.appendBatch(batchMessage(writerEpoch, 1))
61
+ for (let sequence = 2; sequence <= 40; sequence += 1) latest = await sink.appendBatch(batchMessage(writerEpoch, sequence))
62
+ expect(Object.hasOwn(latest, "durableBatchIds")).toBe(false)
63
+ expect(latest.lastDurableBatch).toEqual({
64
+ batchId: batchMessage(writerEpoch, 40).batchId,
65
+ firstSequence: "40",
66
+ lastSequence: "40"
67
+ })
68
+ expect((await sink.appendBatch(batchMessage(writerEpoch, 40))).duplicate).toBe(true)
90
69
  await sink.close()
91
- }
92
- assert.equal(segments.length, 3)
93
- for (let index = 0; index < segments.length; index += 1) {
94
- await utimes(/** @type {string} */ (segments[index]), new Date(1_000 + index), new Date(1_000 + index))
95
- }
96
- await enforceLogRetention({directory: root, maxAgeMs: 1, maxBytes: 1_000_000, now: () => new Date(1_003)})
97
- assert.equal((await readdir(root)).filter((name) => name.endsWith(".jsonl.zst")).length, 1)
98
70
 
99
- const recovered = new CompressedLogWorkerSink(options)
100
- assert.equal((await recovered.start()).durableHighWater, "3")
101
- assert.equal((await recovered.appendBatch(batchMessage(writerEpoch, 4))).durableHighWater, "4")
102
- await recovered.close()
71
+ const restarted = new CompressedLogWorkerSink(options)
72
+ const recovered = await restarted.start()
73
+ expect(Object.hasOwn(recovered, "durableBatchIds")).toBe(false)
74
+ expect(recovered.lastDurableBatch).toEqual(latest.lastDurableBatch)
75
+ await restarted.close()
76
+ }))
103
77
 
104
- const gapDirectory = join(root, "interior-gap")
105
- const gapEpoch = randomBytes(16)
106
- const gapOptions = {...options, directory: gapDirectory, writerEpoch: gapEpoch.toString("base64url")}
107
- const gapSegments = []
108
- for (let sequence = 1; sequence <= 3; sequence += 1) {
109
- const sink = new CompressedLogWorkerSink(gapOptions)
110
- await sink.start()
111
- await sink.appendBatch(batchMessage(gapEpoch, sequence))
112
- gapSegments.push(/** @type {string} */ (sink.status().segmentPath))
113
- await sink.close()
114
- }
115
- const missing = /** @type {string} */ (gapSegments[1])
116
- await Promise.all([rm(missing), rm(`${missing}.idx`), rm(`${missing}.sealed`)])
117
- await assert.rejects(() => new CompressedLogWorkerSink(gapOptions).start(), /interior sequence gap/)
78
+ it("recovery accepts a retention-deleted prefix but rejects interior gaps", async () => withTestCleanup(async (after) => {
79
+ const root = await mkdtemp(join(tmpdir(), "socketduct-compressed-retained-prefix-"))
80
+ after(() => rm(root, {recursive: true, force: true}))
81
+ const writerEpoch = randomBytes(16)
82
+ const options = {directory: root, role: "reverse-hub", writerEpoch: writerEpoch.toString("base64url"), maxSegmentBytes: 4096, compressionLevel: 19, retentionBytes: 1_000_000}
83
+
84
+ const segments = []
85
+ for (let sequence = 1; sequence <= 3; sequence += 1) {
86
+ const sink = new CompressedLogWorkerSink(options)
87
+ await sink.start()
88
+ await sink.appendBatch(batchMessage(writerEpoch, sequence))
89
+ segments.push(/** @type {string} */ (sink.status().segmentPath))
90
+ await sink.close()
91
+ }
92
+ expect(segments.length).toBe(3)
93
+ for (let index = 0; index < segments.length; index += 1) {
94
+ await utimes(/** @type {string} */ (segments[index]), new Date(1_000 + index), new Date(1_000 + index))
95
+ }
96
+ await enforceLogRetention({directory: root, maxAgeMs: 1, maxBytes: 1_000_000, now: () => new Date(1_003)})
97
+ expect((await readdir(root)).filter((name) => name.endsWith(".jsonl.zst")).length).toBe(1)
98
+
99
+ const recovered = new CompressedLogWorkerSink(options)
100
+ expect((await recovered.start()).durableHighWater).toBe("3")
101
+ expect((await recovered.appendBatch(batchMessage(writerEpoch, 4))).durableHighWater).toBe("4")
102
+ await recovered.close()
103
+
104
+ const gapDirectory = join(root, "interior-gap")
105
+ const gapEpoch = randomBytes(16)
106
+ const gapOptions = {...options, directory: gapDirectory, writerEpoch: gapEpoch.toString("base64url")}
107
+ const gapSegments = []
108
+ for (let sequence = 1; sequence <= 3; sequence += 1) {
109
+ const sink = new CompressedLogWorkerSink(gapOptions)
110
+ await sink.start()
111
+ await sink.appendBatch(batchMessage(gapEpoch, sequence))
112
+ gapSegments.push(/** @type {string} */ (sink.status().segmentPath))
113
+ await sink.close()
114
+ }
115
+ const missing = /** @type {string} */ (gapSegments[1])
116
+ await Promise.all([rm(missing), rm(`${missing}.idx`), rm(`${missing}.sealed`)])
117
+ await expect(new CompressedLogWorkerSink(gapOptions).start()).rejects.toThrow(/interior sequence gap/)
118
+ }))
118
119
  })
119
120
 
120
121
  /**