socketduct 0.1.25 → 0.1.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +2 -2
- package/changelog.d/20261010130509-bound-log-frame-compression-memory.md +1 -0
- package/changelog.d/20261025070000-reclaim-unreadable-retention-lock.md +14 -0
- package/docs/runbooks/jsonl-integrity-rollout.md +8 -6
- package/package.json +2 -2
- package/scripts/test.js +9 -14
- package/src/logging/compressed-log-writer.js +1 -1
- package/src/logging/log-retention.js +36 -1
- package/src/logging/zstd-frame-codec.js +15 -1
- package/test/e2e/gateway-process.test.js +93 -92
- package/test/e2e/logger-shutdown-process.test.js +40 -37
- package/test/e2e/reverse-client-recovery-process.test.js +107 -103
- package/test/e2e/reverse-gateway-process.test.js +328 -318
- package/test/integration/initial-target-recovery.test.js +333 -334
- package/test/integration/logging-cli.test.js +121 -119
- package/test/integration/logging-compressed-recovery.test.js +96 -95
- package/test/integration/multiplexed-compatibility.test.js +178 -177
- package/test/integration/multiplexed-corrections.test.js +360 -359
- package/test/integration/multiplexed-failures.test.js +253 -251
- package/test/integration/multiplexed-transport-set.test.js +334 -322
- package/test/integration/multiplexed-transport.test.js +135 -134
- package/test/integration/reverse-gateway-stream-id-reset.test.js +90 -89
- package/test/integration/reverse-gateway.test.js +2309 -2218
- package/test/integration/reverse-resume-continuity.test.js +219 -218
- package/test/integration/reverse-route-class.test.js +358 -341
- package/test/integration/socketduct-http-agent.test.js +314 -312
- package/test/integration/socketduct-https-agent.test.js +516 -485
- package/test/integration/stream-compression.test.js +862 -824
- package/test/integration/tcp-forwarding.test.js +2550 -2447
- package/test/integration/tls-forwarding.test.js +593 -592
- package/test/unit/auth/auth.test.js +43 -44
- package/test/unit/auth/static-token-store.test.js +9 -8
- package/test/unit/client/connect.test.js +41 -40
- package/test/unit/client/deferred-virtual-socket.test.js +95 -94
- package/test/unit/client/multiplexed-spool-budget.test.js +155 -154
- package/test/unit/client/multiplexed-transport-set.test.js +830 -748
- package/test/unit/client/socketduct-socket.test.js +768 -771
- package/test/unit/client/virtual-socket.test.js +198 -168
- package/test/unit/config/docker-gateway-files.test.js +224 -224
- package/test/unit/config/environment.test.js +472 -492
- package/test/unit/index.test.js +53 -52
- package/test/unit/logging/compressed-log-worker.test.js +23 -22
- package/test/unit/logging/compressed-log-writer.test.js +155 -154
- package/test/unit/logging/log-retention.test.js +71 -48
- package/test/unit/logging/log-segment-index.test.js +84 -81
- package/test/unit/logging/zstd-frame-codec.test.js +28 -19
- package/test/unit/protocol/compression.test.js +254 -298
- package/test/unit/protocol/frame.test.js +280 -276
- package/test/unit/protocol/v3-compression.test.js +21 -23
- package/test/unit/protocol/v3-frame.test.js +42 -41
- package/test/unit/release-configuration.test.js +15 -12
- package/test/unit/reverse/disconnect-attribution.test.js +65 -64
- package/test/unit/reverse-terminal-outcomes.test.js +53 -50
- package/test/unit/server/control-handler.test.js +232 -231
- package/test/unit/server/primary.test.js +51 -49
- package/test/unit/server/relay-runtime.test.js +280 -294
- package/test/unit/sessions/session.test.js +48 -62
- package/test/unit/spool/spool-file.test.js +401 -385
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
/* eslint-disable jsdoc/escape-inline-tags, jsdoc/require-param, jsdoc/require-returns */
|
|
2
|
-
import
|
|
2
|
+
import {describe, expect, it, withTestCleanup} from "@velocious/testing"
|
|
3
3
|
import {execFile, spawn} from "node:child_process"
|
|
4
4
|
import {mkdtemp, readFile, readdir, rm, utimes, writeFile} from "node:fs/promises"
|
|
5
5
|
import {tmpdir} from "node:os"
|
|
6
6
|
import {join} from "node:path"
|
|
7
|
-
import test from "node:test"
|
|
8
7
|
import {promisify} from "node:util"
|
|
9
8
|
|
|
10
9
|
import {createCompressedJsonlFileLogger} from "../../src/logging/compressed-log-writer.js"
|
|
@@ -14,127 +13,130 @@ import {enforceLogRetention} from "../../src/logging/log-retention.js"
|
|
|
14
13
|
const run = promisify(execFile)
|
|
15
14
|
const cli = new URL("../../src/cli/socketduct.js", import.meta.url)
|
|
16
15
|
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
16
|
+
describe("socketduct logs CLI", () => {
|
|
17
|
+
it("owned CLI searches and tails durable compressed records with stable exits", async () => withTestCleanup(async (after) => {
|
|
18
|
+
const root = await mkdtemp(join(tmpdir(), "socketduct-logs-cli-"))
|
|
19
|
+
const directory = join(root, "logs")
|
|
20
|
+
after(() => rm(root, {recursive: true, force: true}))
|
|
21
|
+
const logger = createCompressedJsonlFileLogger({directory, role: "reverse-hub"})
|
|
22
|
+
logger.log("process.started", {runtimeMode: "reverse-hub"})
|
|
23
|
+
logger.log("runtime.status", {virtualStreams: 7})
|
|
24
|
+
await logger.close()
|
|
25
|
+
|
|
26
|
+
const match = await invoke("logs", "search", "--directory", directory, "--event", "runtime.status")
|
|
27
|
+
expect(match.code).toBe(0)
|
|
28
|
+
expect(match.stdout.trim().split("\n").map((line) => JSON.parse(line).event)).toEqual(["runtime.status"])
|
|
29
|
+
|
|
30
|
+
const absent = await invoke("logs", "search", "--directory", directory, "--event", "transport.failed")
|
|
31
|
+
expect(absent.code).toBe(1)
|
|
32
|
+
expect(absent.stdout).toBe("")
|
|
33
|
+
|
|
34
|
+
const tail = await invoke("logs", "tail", "--directory", directory)
|
|
35
|
+
expect(tail.code).toBe(0)
|
|
36
|
+
expect(tail.stdout.trim().split("\n").map((line) => JSON.parse(line).event)).toEqual(["process.started", "runtime.status", "log_batch_accepted", "log_frame_committed", "log_closed"])
|
|
37
|
+
|
|
38
|
+
const invalid = await invoke("logs", "search", "--directory", directory, "--client-generation", "not-a-number")
|
|
39
|
+
expect(invalid.code).toBe(2)
|
|
40
|
+
expect(invalid.stderr).toMatch(/Socketduct logs error/)
|
|
41
|
+
}))
|
|
42
|
+
|
|
43
|
+
it("owned CLI follow stays live and emits newly durable writer records", async () => withTestCleanup(async (after) => {
|
|
44
|
+
const root = await mkdtemp(join(tmpdir(), "socketduct-logs-follow-"))
|
|
45
|
+
const directory = join(root, "logs")
|
|
46
|
+
after(() => rm(root, {recursive: true, force: true}))
|
|
47
|
+
const initial = createCompressedJsonlFileLogger({directory, role: "reverse-client"})
|
|
48
|
+
initial.log("process.started", {runtimeMode: "reverse-client"})
|
|
49
|
+
await initial.close()
|
|
50
|
+
|
|
51
|
+
const child = spawn(process.execPath, [cli.pathname, "logs", "tail", "--directory", directory, "--follow", "--poll-ms", "10"], {stdio: ["ignore", "pipe", "pipe"]})
|
|
52
|
+
let stdout = ""
|
|
53
|
+
let stderr = ""
|
|
54
|
+
child.stdout.on("data", (chunk) => { stdout += chunk.toString() })
|
|
55
|
+
child.stderr.on("data", (chunk) => { stderr += chunk.toString() })
|
|
56
|
+
after(() => { if (child.exitCode === null) child.kill("SIGKILL") })
|
|
57
|
+
await waitFor(() => stdout.includes('"eventSequence":"1"'), child, () => stderr)
|
|
58
|
+
|
|
59
|
+
const later = createCompressedJsonlFileLogger({directory, role: "reverse-client"})
|
|
60
|
+
later.log("runtime.status", {virtualStreams: 2})
|
|
61
|
+
await later.close()
|
|
62
|
+
await waitFor(() => stdout.includes('"event":"runtime.status"'), child, () => stderr)
|
|
63
|
+
child.kill("SIGTERM")
|
|
64
|
+
const code = await new Promise((resolve) => child.once("exit", resolve))
|
|
65
|
+
expect(code).toBe(0)
|
|
66
|
+
}))
|
|
67
|
+
|
|
68
|
+
it("owned CLI exits 2 instead of hiding corrupt interior evidence", async () => withTestCleanup(async (after) => {
|
|
69
|
+
const root = await mkdtemp(join(tmpdir(), "socketduct-logs-corrupt-"))
|
|
70
|
+
const directory = join(root, "logs")
|
|
71
|
+
after(() => rm(root, {recursive: true, force: true}))
|
|
72
|
+
const logger = createCompressedJsonlFileLogger({directory, role: "reverse-gateway"})
|
|
73
|
+
logger.log("process.started", {runtimeMode: "reverse-gateway"})
|
|
74
|
+
await logger.close()
|
|
75
|
+
const name = (await readdir(directory)).find((entry) => entry.endsWith(".jsonl.zst"))
|
|
76
|
+
expect(name).toBeDefined()
|
|
77
|
+
const path = join(directory, /** @type {string} */ (name))
|
|
78
|
+
const bytes = await readFile(path)
|
|
79
|
+
bytes[bytes.length - 1] = /** @type {number} */ (bytes[bytes.length - 1]) ^ 0xff
|
|
80
|
+
await writeFile(path, bytes)
|
|
81
|
+
|
|
82
|
+
const result = await invoke("logs", "search", "--directory", directory, "--event", "process.started")
|
|
83
|
+
expect(result.code).toBe(2)
|
|
84
|
+
expect(result.stdout).toBe("")
|
|
85
|
+
expect(result.stderr).toMatch(/Socketduct log evidence error/)
|
|
86
|
+
}))
|
|
87
|
+
|
|
88
|
+
it("tail reader incrementally follows rotation and prunes retention-deleted cursors", async () => withTestCleanup(async (after) => {
|
|
89
|
+
const root = await mkdtemp(join(tmpdir(), "socketduct-logs-incremental-tail-"))
|
|
90
|
+
const directory = join(root, "logs")
|
|
91
|
+
after(() => rm(root, {recursive: true, force: true}))
|
|
92
|
+
const logger = createCompressedJsonlFileLogger({directory, role: "reverse-hub", frameMaxBytes: 256, maxSegmentBytes: 1024, compressionLevel: 1})
|
|
93
|
+
await logger.flush()
|
|
94
|
+
for (let index = 0; index < 8; index += 1) {
|
|
95
|
+
logger.log("process.started", {runtimeMode: "reverse-hub"})
|
|
96
|
+
await logger.flush()
|
|
97
|
+
}
|
|
98
|
+
await logger.flush()
|
|
67
99
|
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
const path = join(directory, name)
|
|
78
|
-
const bytes = await readFile(path)
|
|
79
|
-
bytes[bytes.length - 1] = /** @type {number} */ (bytes[bytes.length - 1]) ^ 0xff
|
|
80
|
-
await writeFile(path, bytes)
|
|
81
|
-
|
|
82
|
-
const result = await invoke("logs", "search", "--directory", directory, "--event", "process.started")
|
|
83
|
-
assert.equal(result.code, 2)
|
|
84
|
-
assert.equal(result.stdout, "")
|
|
85
|
-
assert.match(result.stderr, /Socketduct log evidence error/)
|
|
86
|
-
})
|
|
100
|
+
const reader = new CompressedLogTailReader(directory)
|
|
101
|
+
const initial = await reader.poll()
|
|
102
|
+
expect(initial.records.length > 0).toBeTruthy()
|
|
103
|
+
const identities = new Set(initial.records.map((record) => `${record.writerEpoch}\0${record.eventSequence}`))
|
|
104
|
+
expect(identities.size).toBe(initial.records.length)
|
|
105
|
+
const decoded = reader.status().decodedFrames
|
|
106
|
+
expect(decoded > 0).toBeTruthy()
|
|
107
|
+
expect((await reader.poll()).records).toEqual([])
|
|
108
|
+
expect(reader.status().decodedFrames).toBe(decoded)
|
|
87
109
|
|
|
88
|
-
test("tail reader incrementally follows rotation and prunes retention-deleted cursors", async (t) => {
|
|
89
|
-
const root = await mkdtemp(join(tmpdir(), "socketduct-logs-incremental-tail-"))
|
|
90
|
-
const directory = join(root, "logs")
|
|
91
|
-
t.after(() => rm(root, {recursive: true, force: true}))
|
|
92
|
-
const logger = createCompressedJsonlFileLogger({directory, role: "reverse-hub", frameMaxBytes: 256, maxSegmentBytes: 1024, compressionLevel: 1})
|
|
93
|
-
await logger.flush()
|
|
94
|
-
for (let index = 0; index < 8; index += 1) {
|
|
95
110
|
logger.log("process.started", {runtimeMode: "reverse-hub"})
|
|
96
111
|
await logger.flush()
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
later.log("runtime.status", {virtualStreams: 2})
|
|
126
|
-
await later.close()
|
|
127
|
-
const laterSegments = (await readdir(directory)).filter((name) => name.endsWith(".jsonl.zst") && !oldSegments.includes(name))
|
|
128
|
-
for (const name of laterSegments) await utimes(join(directory, name), new Date(10_000), new Date(10_000))
|
|
129
|
-
const laterDelta = await reader.poll()
|
|
130
|
-
assert(laterDelta.records.some((record) => record.event === "runtime.status"))
|
|
131
|
-
|
|
132
|
-
await enforceLogRetention({directory, maxAgeMs: 5_000, maxBytes: 1_000_000, now: () => new Date(10_000)})
|
|
133
|
-
const beforePruneDecode = reader.status().decodedFrames
|
|
134
|
-
assert.deepEqual((await reader.poll()).records, [])
|
|
135
|
-
const survivingSegments = (await readdir(directory)).filter((name) => name.endsWith(".jsonl.zst")).length
|
|
136
|
-
assert(reader.status().trackedSegments <= survivingSegments)
|
|
137
|
-
assert.equal(reader.status().decodedFrames, beforePruneDecode)
|
|
112
|
+
await logger.flush()
|
|
113
|
+
const activeDelta = await reader.poll()
|
|
114
|
+
expect(activeDelta.records.length > 0).toBeTruthy()
|
|
115
|
+
for (const record of activeDelta.records) expect(identities.has(`${record.writerEpoch}\0${record.eventSequence}`)).toBe(false)
|
|
116
|
+
await logger.close()
|
|
117
|
+
const closeDelta = await reader.poll()
|
|
118
|
+
expect(closeDelta.records.length > 0).toBeTruthy()
|
|
119
|
+
for (const record of closeDelta.records) expect(identities.has(`${record.writerEpoch}\0${record.eventSequence}`)).toBe(false)
|
|
120
|
+
|
|
121
|
+
const oldSegments = (await readdir(directory)).filter((name) => name.endsWith(".jsonl.zst"))
|
|
122
|
+
// fixture must rotate through more than one segment
|
|
123
|
+
expect(oldSegments.length > 1).toBeTruthy()
|
|
124
|
+
for (const name of oldSegments) await utimes(join(directory, name), new Date(1_000), new Date(1_000))
|
|
125
|
+
const later = createCompressedJsonlFileLogger({directory, role: "reverse-hub", compressionLevel: 1})
|
|
126
|
+
later.log("runtime.status", {virtualStreams: 2})
|
|
127
|
+
await later.close()
|
|
128
|
+
const laterSegments = (await readdir(directory)).filter((name) => name.endsWith(".jsonl.zst") && !oldSegments.includes(name))
|
|
129
|
+
for (const name of laterSegments) await utimes(join(directory, name), new Date(10_000), new Date(10_000))
|
|
130
|
+
const laterDelta = await reader.poll()
|
|
131
|
+
expect(laterDelta.records.some((record) => record.event === "runtime.status")).toBeTruthy()
|
|
132
|
+
|
|
133
|
+
await enforceLogRetention({directory, maxAgeMs: 5_000, maxBytes: 1_000_000, now: () => new Date(10_000)})
|
|
134
|
+
const beforePruneDecode = reader.status().decodedFrames
|
|
135
|
+
expect((await reader.poll()).records).toEqual([])
|
|
136
|
+
const survivingSegments = (await readdir(directory)).filter((name) => name.endsWith(".jsonl.zst")).length
|
|
137
|
+
expect(reader.status().trackedSegments <= survivingSegments).toBeTruthy()
|
|
138
|
+
expect(reader.status().decodedFrames).toBe(beforePruneDecode)
|
|
139
|
+
}))
|
|
138
140
|
})
|
|
139
141
|
|
|
140
142
|
/** @param {...string} args - CLI arguments. */
|
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
/* eslint-disable jsdoc/escape-inline-tags, jsdoc/require-param-type, jsdoc/require-returns */
|
|
2
|
-
import
|
|
2
|
+
import {describe, expect, it, withTestCleanup} from "@velocious/testing"
|
|
3
3
|
import {createHash, randomBytes} from "node:crypto"
|
|
4
4
|
import {mkdtemp, readdir, rm, stat, utimes, writeFile} from "node:fs/promises"
|
|
5
5
|
import {tmpdir} from "node:os"
|
|
6
6
|
import {join} from "node:path"
|
|
7
|
-
import test from "node:test"
|
|
8
7
|
|
|
9
8
|
import {createLogBatchId, inspectLogBatch} from "../../src/logging/log-batch.js"
|
|
10
9
|
import {readCompressedLogDirectory} from "../../src/logging/compressed-log-reader.js"
|
|
@@ -13,108 +12,110 @@ import {enforceLogRetention} from "../../src/logging/log-retention.js"
|
|
|
13
12
|
import {createIndexEntry, createIndexHeader, INDEX_ENTRY_BYTES, INDEX_HEADER_BYTES, uuidBytes} from "../../src/logging/log-segment-index.js"
|
|
14
13
|
import {compressLogFrame} from "../../src/logging/zstd-frame-codec.js"
|
|
15
14
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
15
|
+
describe("compressed log recovery", () => {
|
|
16
|
+
it("recovery indexes a complete data-synced orphan without duplicating its batch", async () => withTestCleanup(async (after) => {
|
|
17
|
+
const root = await mkdtemp(join(tmpdir(), "socketduct-compressed-orphan-"))
|
|
18
|
+
after(() => rm(root, {recursive: true, force: true}))
|
|
19
|
+
const fixture = await createRecoveryFixture(root, false)
|
|
20
|
+
const sink = new CompressedLogWorkerSink(fixture.options)
|
|
21
|
+
const recovered = await sink.start()
|
|
22
|
+
expect(recovered.durableHighWater).toBe("2")
|
|
23
|
+
expect(recovered.orphanRecoveredCount).toBe(1)
|
|
24
|
+
expect(recovered.lastDurableBatch?.batchId).toBe(fixture.secondBatchId)
|
|
25
|
+
expect((await stat(fixture.indexPath)).size).toBe(INDEX_HEADER_BYTES + 2 * INDEX_ENTRY_BYTES)
|
|
26
26
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
})
|
|
35
|
-
|
|
36
|
-
test("recovery truncates only a torn final data frame and accepts its replay", async (t) => {
|
|
37
|
-
const root = await mkdtemp(join(tmpdir(), "socketduct-compressed-torn-"))
|
|
38
|
-
t.after(() => rm(root, {recursive: true, force: true}))
|
|
39
|
-
const fixture = await createRecoveryFixture(root, true)
|
|
40
|
-
const sink = new CompressedLogWorkerSink(fixture.options)
|
|
41
|
-
const recovered = await sink.start()
|
|
42
|
-
assert.equal(recovered.durableHighWater, "1")
|
|
43
|
-
assert.equal((await stat(fixture.dataPath)).size, fixture.firstFrame.length)
|
|
44
|
-
|
|
45
|
-
const replayed = await sink.appendBatch(fixture.secondMessage)
|
|
46
|
-
assert.equal(replayed.duplicate, false)
|
|
47
|
-
await sink.close()
|
|
48
|
-
const result = await readCompressedLogDirectory(root)
|
|
49
|
-
assert.equal(result.incomplete, false)
|
|
50
|
-
assert.deepEqual(result.records.map((record) => record.eventSequence), ["1", "2"])
|
|
51
|
-
})
|
|
52
|
-
|
|
53
|
-
test("worker acknowledgements retain only the latest durable batch identity", async (t) => {
|
|
54
|
-
const root = await mkdtemp(join(tmpdir(), "socketduct-compressed-bounded-ack-"))
|
|
55
|
-
t.after(() => rm(root, {recursive: true, force: true}))
|
|
56
|
-
const writerEpoch = randomBytes(16)
|
|
57
|
-
const options = {directory: root, role: "reverse-hub", writerEpoch: writerEpoch.toString("base64url"), maxSegmentBytes: 4096, compressionLevel: 19}
|
|
58
|
-
const sink = new CompressedLogWorkerSink(options)
|
|
59
|
-
await sink.start()
|
|
60
|
-
let latest = await sink.appendBatch(batchMessage(writerEpoch, 1))
|
|
61
|
-
for (let sequence = 2; sequence <= 40; sequence += 1) latest = await sink.appendBatch(batchMessage(writerEpoch, sequence))
|
|
62
|
-
assert.equal(Object.hasOwn(latest, "durableBatchIds"), false)
|
|
63
|
-
assert.deepEqual(latest.lastDurableBatch, {
|
|
64
|
-
batchId: batchMessage(writerEpoch, 40).batchId,
|
|
65
|
-
firstSequence: "40",
|
|
66
|
-
lastSequence: "40"
|
|
67
|
-
})
|
|
68
|
-
assert.equal((await sink.appendBatch(batchMessage(writerEpoch, 40))).duplicate, true)
|
|
69
|
-
await sink.close()
|
|
27
|
+
const duplicate = await sink.appendBatch(fixture.secondMessage)
|
|
28
|
+
expect(duplicate.duplicate).toBe(true)
|
|
29
|
+
expect((await stat(fixture.dataPath)).size).toBe(fixture.firstFrame.length + fixture.secondFrame.length)
|
|
30
|
+
await sink.close()
|
|
31
|
+
const result = await readCompressedLogDirectory(root)
|
|
32
|
+
expect(result.incomplete).toBe(false)
|
|
33
|
+
expect(result.records.map((record) => record.eventSequence)).toEqual(["1", "2"])
|
|
34
|
+
}))
|
|
70
35
|
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
36
|
+
it("recovery truncates only a torn final data frame and accepts its replay", async () => withTestCleanup(async (after) => {
|
|
37
|
+
const root = await mkdtemp(join(tmpdir(), "socketduct-compressed-torn-"))
|
|
38
|
+
after(() => rm(root, {recursive: true, force: true}))
|
|
39
|
+
const fixture = await createRecoveryFixture(root, true)
|
|
40
|
+
const sink = new CompressedLogWorkerSink(fixture.options)
|
|
41
|
+
const recovered = await sink.start()
|
|
42
|
+
expect(recovered.durableHighWater).toBe("1")
|
|
43
|
+
expect((await stat(fixture.dataPath)).size).toBe(fixture.firstFrame.length)
|
|
77
44
|
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
45
|
+
const replayed = await sink.appendBatch(fixture.secondMessage)
|
|
46
|
+
expect(replayed.duplicate).toBe(false)
|
|
47
|
+
await sink.close()
|
|
48
|
+
const result = await readCompressedLogDirectory(root)
|
|
49
|
+
expect(result.incomplete).toBe(false)
|
|
50
|
+
expect(result.records.map((record) => record.eventSequence)).toEqual(["1", "2"])
|
|
51
|
+
}))
|
|
83
52
|
|
|
84
|
-
|
|
85
|
-
|
|
53
|
+
it("worker acknowledgements retain only the latest durable batch identity", async () => withTestCleanup(async (after) => {
|
|
54
|
+
const root = await mkdtemp(join(tmpdir(), "socketduct-compressed-bounded-ack-"))
|
|
55
|
+
after(() => rm(root, {recursive: true, force: true}))
|
|
56
|
+
const writerEpoch = randomBytes(16)
|
|
57
|
+
const options = {directory: root, role: "reverse-hub", writerEpoch: writerEpoch.toString("base64url"), maxSegmentBytes: 4096, compressionLevel: 19}
|
|
86
58
|
const sink = new CompressedLogWorkerSink(options)
|
|
87
59
|
await sink.start()
|
|
88
|
-
await sink.appendBatch(batchMessage(writerEpoch,
|
|
89
|
-
|
|
60
|
+
let latest = await sink.appendBatch(batchMessage(writerEpoch, 1))
|
|
61
|
+
for (let sequence = 2; sequence <= 40; sequence += 1) latest = await sink.appendBatch(batchMessage(writerEpoch, sequence))
|
|
62
|
+
expect(Object.hasOwn(latest, "durableBatchIds")).toBe(false)
|
|
63
|
+
expect(latest.lastDurableBatch).toEqual({
|
|
64
|
+
batchId: batchMessage(writerEpoch, 40).batchId,
|
|
65
|
+
firstSequence: "40",
|
|
66
|
+
lastSequence: "40"
|
|
67
|
+
})
|
|
68
|
+
expect((await sink.appendBatch(batchMessage(writerEpoch, 40))).duplicate).toBe(true)
|
|
90
69
|
await sink.close()
|
|
91
|
-
}
|
|
92
|
-
assert.equal(segments.length, 3)
|
|
93
|
-
for (let index = 0; index < segments.length; index += 1) {
|
|
94
|
-
await utimes(/** @type {string} */ (segments[index]), new Date(1_000 + index), new Date(1_000 + index))
|
|
95
|
-
}
|
|
96
|
-
await enforceLogRetention({directory: root, maxAgeMs: 1, maxBytes: 1_000_000, now: () => new Date(1_003)})
|
|
97
|
-
assert.equal((await readdir(root)).filter((name) => name.endsWith(".jsonl.zst")).length, 1)
|
|
98
70
|
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
71
|
+
const restarted = new CompressedLogWorkerSink(options)
|
|
72
|
+
const recovered = await restarted.start()
|
|
73
|
+
expect(Object.hasOwn(recovered, "durableBatchIds")).toBe(false)
|
|
74
|
+
expect(recovered.lastDurableBatch).toEqual(latest.lastDurableBatch)
|
|
75
|
+
await restarted.close()
|
|
76
|
+
}))
|
|
103
77
|
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
78
|
+
it("recovery accepts a retention-deleted prefix but rejects interior gaps", async () => withTestCleanup(async (after) => {
|
|
79
|
+
const root = await mkdtemp(join(tmpdir(), "socketduct-compressed-retained-prefix-"))
|
|
80
|
+
after(() => rm(root, {recursive: true, force: true}))
|
|
81
|
+
const writerEpoch = randomBytes(16)
|
|
82
|
+
const options = {directory: root, role: "reverse-hub", writerEpoch: writerEpoch.toString("base64url"), maxSegmentBytes: 4096, compressionLevel: 19, retentionBytes: 1_000_000}
|
|
83
|
+
|
|
84
|
+
const segments = []
|
|
85
|
+
for (let sequence = 1; sequence <= 3; sequence += 1) {
|
|
86
|
+
const sink = new CompressedLogWorkerSink(options)
|
|
87
|
+
await sink.start()
|
|
88
|
+
await sink.appendBatch(batchMessage(writerEpoch, sequence))
|
|
89
|
+
segments.push(/** @type {string} */ (sink.status().segmentPath))
|
|
90
|
+
await sink.close()
|
|
91
|
+
}
|
|
92
|
+
expect(segments.length).toBe(3)
|
|
93
|
+
for (let index = 0; index < segments.length; index += 1) {
|
|
94
|
+
await utimes(/** @type {string} */ (segments[index]), new Date(1_000 + index), new Date(1_000 + index))
|
|
95
|
+
}
|
|
96
|
+
await enforceLogRetention({directory: root, maxAgeMs: 1, maxBytes: 1_000_000, now: () => new Date(1_003)})
|
|
97
|
+
expect((await readdir(root)).filter((name) => name.endsWith(".jsonl.zst")).length).toBe(1)
|
|
98
|
+
|
|
99
|
+
const recovered = new CompressedLogWorkerSink(options)
|
|
100
|
+
expect((await recovered.start()).durableHighWater).toBe("3")
|
|
101
|
+
expect((await recovered.appendBatch(batchMessage(writerEpoch, 4))).durableHighWater).toBe("4")
|
|
102
|
+
await recovered.close()
|
|
103
|
+
|
|
104
|
+
const gapDirectory = join(root, "interior-gap")
|
|
105
|
+
const gapEpoch = randomBytes(16)
|
|
106
|
+
const gapOptions = {...options, directory: gapDirectory, writerEpoch: gapEpoch.toString("base64url")}
|
|
107
|
+
const gapSegments = []
|
|
108
|
+
for (let sequence = 1; sequence <= 3; sequence += 1) {
|
|
109
|
+
const sink = new CompressedLogWorkerSink(gapOptions)
|
|
110
|
+
await sink.start()
|
|
111
|
+
await sink.appendBatch(batchMessage(gapEpoch, sequence))
|
|
112
|
+
gapSegments.push(/** @type {string} */ (sink.status().segmentPath))
|
|
113
|
+
await sink.close()
|
|
114
|
+
}
|
|
115
|
+
const missing = /** @type {string} */ (gapSegments[1])
|
|
116
|
+
await Promise.all([rm(missing), rm(`${missing}.idx`), rm(`${missing}.sealed`)])
|
|
117
|
+
await expect(new CompressedLogWorkerSink(gapOptions).start()).rejects.toThrow(/interior sequence gap/)
|
|
118
|
+
}))
|
|
118
119
|
})
|
|
119
120
|
|
|
120
121
|
/**
|