socketduct 0.1.18 → 0.1.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/.env.example +17 -5
  2. package/README.md +5 -5
  3. package/changelog.d/20260925143000-reverse-client-process-recovery.md +1 -0
  4. package/changelog.d/20260925170000-disconnect-attribution-compressed-logs.md +1 -0
  5. package/docs/architecture.md +11 -9
  6. package/docs/docker.md +5 -4
  7. package/docs/logging.md +131 -159
  8. package/docs/protocol.md +43 -7
  9. package/docs/reverse-gateway.md +75 -9
  10. package/docs/running.md +16 -13
  11. package/docs/security.md +16 -5
  12. package/docs/session-lifecycle.md +14 -3
  13. package/docs/socket-api.md +54 -20
  14. package/package.json +5 -1
  15. package/scripts/verify-zstd-interoperability.mjs +41 -0
  16. package/src/cli/socketduct.js +143 -0
  17. package/src/client/multiplexed-transport-set.js +12 -5
  18. package/src/client/virtual-socket.js +52 -0
  19. package/src/config/environment.js +56 -7
  20. package/src/index.js +64 -26
  21. package/src/logging/compressed-log-reader.js +256 -0
  22. package/src/logging/compressed-log-worker.js +430 -0
  23. package/src/logging/compressed-log-writer.js +538 -0
  24. package/src/logging/log-batch.js +72 -0
  25. package/src/logging/log-retention.js +241 -0
  26. package/src/logging/log-segment-index.js +176 -0
  27. package/src/logging/structured-logger.js +136 -30
  28. package/src/logging/zstd-frame-codec.js +73 -0
  29. package/src/reverse/client.js +372 -82
  30. package/src/reverse/disconnect-attribution.js +191 -0
  31. package/src/reverse/gateway.js +68 -18
  32. package/src/reverse/hub.js +215 -32
  33. package/src/reverse/runtime.js +5 -3
  34. package/src/spool/spool-file.js +136 -7
  35. package/tensorbuzz.yml +4 -2
  36. package/test/e2e/logger-shutdown-process.test.js +86 -0
  37. package/test/e2e/reverse-client-recovery-process.test.js +170 -0
  38. package/test/e2e/reverse-gateway-process.test.js +15 -14
  39. package/test/fixtures/reverse-client-recovery.mjs +123 -0
  40. package/test/integration/logging-cli.test.js +168 -0
  41. package/test/integration/logging-compressed-recovery.test.js +186 -0
  42. package/test/integration/reverse-gateway.test.js +597 -1
  43. package/test/integration/reverse-route-class.test.js +6 -1
  44. package/test/unit/client/multiplexed-transport-set.test.js +12 -1
  45. package/test/unit/config/docker-gateway-files.test.js +2 -0
  46. package/test/unit/config/environment.test.js +54 -14
  47. package/test/unit/index.test.js +3 -0
  48. package/test/unit/logging/compressed-log-worker.test.js +29 -0
  49. package/test/unit/logging/compressed-log-writer.test.js +183 -0
  50. package/test/unit/logging/log-retention.test.js +86 -0
  51. package/test/unit/logging/log-segment-index.test.js +104 -0
  52. package/test/unit/logging/structured-logger.test.js +14 -1
  53. package/test/unit/logging/zstd-frame-codec.test.js +24 -0
  54. package/test/unit/reverse/disconnect-attribution.test.js +82 -0
  55. package/test/unit/spool/spool-file.test.js +133 -0
@@ -289,6 +289,17 @@ test("route-class members are first-class reverse transports with route-class al
289
289
  assert(connectionIds.every((connectionId) => typeof connectionId === "string"))
290
290
  assert.equal(new Set(connectionIds).size, 2)
291
291
 
292
+ assert.throws(() => set.recoverConnection(/** @type {Parameters<MultiplexedSocketductTransportSet['recoverConnection']>[0]} */ (/** @type {unknown} */ ({host: "target.internal", port: 443, identifier: "route_stream_01"}))), /exactly one.*gatewayId.*routeClass/i)
293
+ assert.throws(() => set.recoverConnection(/** @type {Parameters<MultiplexedSocketductTransportSet['recoverConnection']>[0]} */ (/** @type {unknown} */ ({gatewayId: "starlink", routeClass: "zyxel", host: "target.internal", port: 443, identifier: "route_stream_01"}))), /exactly one.*gatewayId.*routeClass/i)
294
+ assert.throws(() => set.recoverConnection({routeClass: "missing", host: "target.internal", port: 443, identifier: "route_stream_01"}), /unknown routeClass/i)
295
+ const recovered = set.recoverConnection({routeClass: "zyxel", host: "target.internal", port: 443, identifier: "route_stream_01", sessionName: "durable-route"})
296
+ assert.equal(/** @type {{gatewayId?: string}} */ (recovered).gatewayId, undefined)
297
+ assert.equal(/** @type {{routeClass?: string}} */ (recovered).routeClass, "zyxel")
298
+ assert.equal(Object.getOwnPropertyDescriptor(recovered, "routeClass")?.writable, false)
299
+ assert.deepEqual(members.map((member) => member.opens), [0, 1])
300
+ assert.equal(/** @type {{recover?: unknown}} */ (at(members, 1).recoveryConnectionCalls[0]).recover, true)
301
+ assert.equal(Object.hasOwn(/** @type {object} */ (at(members, 1).recoveryConnectionCalls[0]), "routeClass"), false)
302
+
292
303
  const started = set.start()
293
304
  at(members, 0).ready()
294
305
  await started
@@ -845,7 +856,7 @@ test("pending socket end reserves and sends DataView bytes", async () => {
845
856
  await set.close()
846
857
  })
847
858
 
848
- test("recovery requires an original configured gateway and registration before start", async () => {
859
+ test("recovery requires one original configured selector and registration before start", async () => {
849
860
  const {set, members} = fixture()
850
861
  assert.throws(() => set.recoverConnection(/** @type {Parameters<MultiplexedSocketductTransportSet['recoverConnection']>[0]} */ (/** @type {unknown} */ ({host: "target.internal", port: 443, identifier: "stable_stream_01"}))), /gatewayId/i)
851
862
  assert.throws(() => set.recoverConnection({gatewayId: "unknown", host: "target.internal", port: 443, identifier: "stable_stream_01"}), /unknown gatewayId/i)
@@ -36,6 +36,8 @@ test("TensorBuzz exclusively owns verification and container build validation",
36
36
  const packageMetadata = JSON.parse(packageJson)
37
37
 
38
38
  assert.deepEqual(workflowNames, [])
39
+ assert.match(tensorbuzz, /^dependency_roots:\n {2}- path: \.$/m)
40
+ assert.doesNotMatch(tensorbuzz, /^\s*- npm ci$/m)
39
41
  assert.match(tensorbuzz, /DOCKER_HOST: "tcp:\/\/docker_server:2375"/)
40
42
  assert.match(tensorbuzz, /docker_server:\n\s+image: "docker:[^"]+-dind"/)
41
43
  assert.match(tensorbuzz, /privileged: true/)
@@ -79,37 +79,77 @@ test("loadEnvironmentConfig parses or disables the status interval", () => {
79
79
  assert.throws(() => loadEnvironmentConfig({SOCKETDUCT_STATUS_INTERVAL_MS: "-1"}), /non-negative integer/)
80
80
  })
81
81
 
82
- test("loadEnvironmentConfig keeps persistent logging disabled by default and parses explicit bounds", () => {
82
+ test("loadEnvironmentConfig keeps persistent logging disabled by default and parses compressed bounds", () => {
83
83
  assert.equal(loadEnvironmentConfig({}).fileLogging, undefined)
84
84
  assert.equal(loadEnvironmentConfig({SOCKETDUCT_LOG_DIRECTORY: " "}).fileLogging, undefined)
85
85
  assert.deepEqual(loadEnvironmentConfig({
86
86
  SOCKETDUCT_LOG_DIRECTORY: " /var/log/socketduct ",
87
- SOCKETDUCT_LOG_MAX_FILE_BYTES: "2097152",
88
- SOCKETDUCT_LOG_MAX_FILES: "4"
87
+ SOCKETDUCT_LOG_FRAME_MAX_BYTES: "2097152",
88
+ SOCKETDUCT_LOG_FLUSH_INTERVAL_MS: "1500",
89
+ SOCKETDUCT_LOG_CRITICAL_FLUSH_INTERVAL_MS: "200",
90
+ SOCKETDUCT_LOG_SEGMENT_MAX_BYTES: "268435456",
91
+ SOCKETDUCT_LOG_RETENTION_DAYS: "14",
92
+ SOCKETDUCT_LOG_RETENTION_MAX_BYTES: "3221225472",
93
+ SOCKETDUCT_LOG_QUEUE_MAX_BYTES: "33554432",
94
+ SOCKETDUCT_LOG_CRITICAL_RESERVE_BYTES: "4194304",
95
+ SOCKETDUCT_LOG_COMPRESSION_LEVEL: "18"
89
96
  }).fileLogging, {
90
97
  directory: "/var/log/socketduct",
91
- maxFileBytes: 2_097_152,
92
- maxFiles: 4
98
+ frameMaxBytes: 2_097_152,
99
+ flushIntervalMs: 1_500,
100
+ criticalFlushIntervalMs: 200,
101
+ maxSegmentBytes: 268_435_456,
102
+ retentionDays: 14,
103
+ retentionBytes: 3_221_225_472,
104
+ maxQueueBytes: 33_554_432,
105
+ criticalReserveBytes: 4_194_304,
106
+ compressionLevel: 18
93
107
  })
94
108
  })
95
109
 
96
- test("loadEnvironmentConfig rejects unsafe persistent logging bounds", () => {
110
+ test("loadEnvironmentConfig rejects unsafe compressed logging bounds", () => {
97
111
  assert.throws(() => loadEnvironmentConfig({
98
112
  SOCKETDUCT_LOG_DIRECTORY: "/var/log/socketduct",
99
- SOCKETDUCT_LOG_MAX_FILE_BYTES: "100"
100
- }), /SOCKETDUCT_LOG_MAX_FILE_BYTES/)
113
+ SOCKETDUCT_LOG_FRAME_MAX_BYTES: "100"
114
+ }), /SOCKETDUCT_LOG_FRAME_MAX_BYTES/)
101
115
  assert.throws(() => loadEnvironmentConfig({
102
116
  SOCKETDUCT_LOG_DIRECTORY: "/var/log/socketduct",
103
- SOCKETDUCT_LOG_MAX_FILE_BYTES: "9007199254740992"
104
- }), /SOCKETDUCT_LOG_MAX_FILE_BYTES/)
117
+ SOCKETDUCT_LOG_COMPRESSION_LEVEL: "23"
118
+ }), /SOCKETDUCT_LOG_COMPRESSION_LEVEL/)
119
+ assert.throws(() => loadEnvironmentConfig({
120
+ SOCKETDUCT_LOG_DIRECTORY: "/var/log/socketduct",
121
+ SOCKETDUCT_LOG_QUEUE_MAX_BYTES: "1024",
122
+ SOCKETDUCT_LOG_CRITICAL_RESERVE_BYTES: "1024"
123
+ }), /SOCKETDUCT_LOG_CRITICAL_RESERVE_BYTES/)
105
124
  assert.throws(() => loadEnvironmentConfig({
106
125
  SOCKETDUCT_LOG_DIRECTORY: "/var/log/socketduct",
107
- SOCKETDUCT_LOG_MAX_FILES: "0"
108
- }), /SOCKETDUCT_LOG_MAX_FILES/)
126
+ SOCKETDUCT_LOG_SEGMENT_MAX_BYTES: "5368709120",
127
+ SOCKETDUCT_LOG_RETENTION_MAX_BYTES: "4294967296"
128
+ }), /SOCKETDUCT_LOG_SEGMENT_MAX_BYTES/)
109
129
  assert.throws(() => loadEnvironmentConfig({
110
130
  SOCKETDUCT_LOG_DIRECTORY: "/var/log/socketduct",
111
- SOCKETDUCT_LOG_MAX_FILES: "101"
112
- }), /SOCKETDUCT_LOG_MAX_FILES/)
131
+ SOCKETDUCT_LOG_MAX_FILE_BYTES: "10485760"
132
+ }), /SOCKETDUCT_LOG_MAX_FILE_BYTES/)
133
+ })
134
+
135
+ test("loadEnvironmentConfig accepts a paired hub-only fingerprint key and epoch", () => {
136
+ const encodedKey = Buffer.alloc(32, 0x5a).toString("base64url")
137
+ const config = loadEnvironmentConfig({
138
+ SOCKETDUCT_MODE: "reverse-hub",
139
+ SOCKETDUCT_LOG_DIRECTORY: "/var/log/socketduct",
140
+ SOCKETDUCT_CONNECTION_FINGERPRINT_KEY: encodedKey,
141
+ SOCKETDUCT_CONNECTION_FINGERPRINT_KEY_EPOCH: "hub-2026-09"
142
+ })
143
+ assert.deepEqual(config.connectionFingerprint, {key: Buffer.alloc(32, 0x5a), keyEpoch: "hub-2026-09"})
144
+ assert.throws(() => loadEnvironmentConfig({
145
+ SOCKETDUCT_MODE: "reverse-hub",
146
+ SOCKETDUCT_LOG_DIRECTORY: "/var/log/socketduct"
147
+ }), /SOCKETDUCT_CONNECTION_FINGERPRINT_KEY/)
148
+ assert.throws(() => loadEnvironmentConfig({
149
+ SOCKETDUCT_MODE: "reverse-gateway",
150
+ SOCKETDUCT_CONNECTION_FINGERPRINT_KEY: encodedKey,
151
+ SOCKETDUCT_CONNECTION_FINGERPRINT_KEY_EPOCH: "hub-2026-09"
152
+ }), /hub-only/)
113
153
  })
114
154
 
115
155
  test("loadEnvironmentConfig selects v3 explicitly and keeps v2 as rollback default", () => {
@@ -3,6 +3,7 @@ import test from "node:test"
3
3
 
4
4
  import {
5
5
  connectSocketduct,
6
+ createCompressedJsonlFileLogger,
6
7
  createJsonlFileLogger,
7
8
  createMultiplexedSocketductConnection,
8
9
  createMultiplexedSocketductTransportSet,
@@ -19,6 +20,7 @@ import {createSocketductConnectionFactory as directCreateSocketductConnectionFac
19
20
  import {SocketductHttpAgent as DirectSocketductHttpAgent} from "../../src/client/http-agent.js"
20
21
  import {connectSocketduct as directConnectSocketduct, SocketductSocket as DirectSocketductSocket} from "../../src/client/socketduct-socket.js"
21
22
  import {createJsonlFileLogger as directCreateJsonlFileLogger} from "../../src/logging/structured-logger.js"
23
+ import {createCompressedJsonlFileLogger as directCreateCompressedJsonlFileLogger} from "../../src/logging/compressed-log-writer.js"
22
24
 
23
25
  test("getStartupSummary returns the Socketduct service identity and default mode", () => {
24
26
  assert.deepEqual(getStartupSummary(), {
@@ -33,6 +35,7 @@ test("index exports the public SocketductSocket client API", () => {
33
35
  assert.equal(connectSocketduct, directConnectSocketduct)
34
36
  assert.equal(createSocketductConnectionFactory, directCreateSocketductConnectionFactory)
35
37
  assert.equal(createJsonlFileLogger, directCreateJsonlFileLogger)
38
+ assert.equal(createCompressedJsonlFileLogger, directCreateCompressedJsonlFileLogger)
36
39
  assert.equal(SocketductHttpAgent, DirectSocketductHttpAgent)
37
40
  })
38
41
 
@@ -0,0 +1,29 @@
1
+ import assert from "node:assert/strict"
2
+ import test from "node:test"
3
+
4
+ import {writeAll} from "../../../src/logging/compressed-log-worker.js"
5
+
6
+ test("worker positional writes preserve every byte across all short-write offsets", async () => {
7
+ const source = Buffer.from("short-write-safe-durable-frame")
8
+ for (let maximum = 1; maximum <= source.length; maximum += 1) {
9
+ const destination = Buffer.alloc(source.length + 7)
10
+ /** @type {number[]} */
11
+ const positions = []
12
+ const handle = {
13
+ async write(/** @type {Buffer} */ bytes, /** @type {number} */ offset, /** @type {number} */ length, /** @type {number} */ position) {
14
+ const bytesWritten = Math.min(maximum, length)
15
+ positions.push(position)
16
+ Buffer.from(bytes).copy(destination, position, offset, offset + bytesWritten)
17
+ return {bytesWritten, buffer: bytes}
18
+ }
19
+ }
20
+ await writeAll(/** @type {import('node:fs/promises').FileHandle} */ (handle), source, 7)
21
+ assert.deepEqual(destination.subarray(7), source)
22
+ assert.deepEqual(positions, Array.from({length: Math.ceil(source.length / maximum)}, (_, index) => 7 + index * maximum))
23
+ }
24
+ })
25
+
26
+ test("worker positional writes fail loudly on zero progress", async () => {
27
+ const handle = {async write(/** @type {Buffer} */ bytes) { return {bytesWritten: 0, buffer: bytes} }}
28
+ await assert.rejects(() => writeAll(/** @type {import('node:fs/promises').FileHandle} */ (handle), Buffer.from("frame"), 0), /no progress/)
29
+ })
@@ -0,0 +1,183 @@
1
+ import assert from "node:assert/strict"
2
+ import {mkdtemp, readdir, rm, stat} from "node:fs/promises"
3
+ import {tmpdir} from "node:os"
4
+ import {join} from "node:path"
5
+ import test from "node:test"
6
+
7
+ import {readCompressedLogDirectory} from "../../../src/logging/compressed-log-reader.js"
8
+ import {createCompressedJsonlFileLogger} from "../../../src/logging/compressed-log-writer.js"
9
+
10
+ test("compressed logger durably acknowledges ordered records and closes asynchronously", async (t) => {
11
+ const root = await mkdtemp(join(tmpdir(), "socketduct-compressed-writer-"))
12
+ const directory = join(root, "logs")
13
+ let tick = 0
14
+ const logger = createCompressedJsonlFileLogger({
15
+ directory,
16
+ role: "reverse-hub",
17
+ now: () => new Date(1_795_000_000_000 + tick++),
18
+ frameMaxBytes: 256,
19
+ flushIntervalMs: 60_000,
20
+ criticalFlushIntervalMs: 60_000,
21
+ maxSegmentBytes: 2048
22
+ })
23
+ t.after(async () => { await logger.close().catch(() => {}); await rm(root, {recursive: true, force: true}) })
24
+
25
+ assert.equal(logger.log("process.started", {runtimeMode: "reverse-hub", token: "must-not-survive"}), "accepted")
26
+ assert.equal(logger.log("runtime.status", {virtualStreams: 4}), "accepted")
27
+ assert.equal(logger.log("runtime.status", {virtualStreams: 5}), "accepted")
28
+ assert.equal(logger.status().acceptedHighWater, "3")
29
+ await logger.flush()
30
+
31
+ const durable = logger.status()
32
+ assert(Number(durable.durableHighWater) >= 3)
33
+ assert.equal(durable.health, "healthy")
34
+ const result = await readCompressedLogDirectory(directory)
35
+ assert.equal(result.incomplete, false)
36
+ assert.deepEqual(result.records.map((record) => record.eventSequence), Array.from({length: result.records.length}, (_, index) => String(index + 1)))
37
+ assert.deepEqual(result.records.filter((record) => !String(record.event).startsWith("log_")).map((record) => record.event), ["process.started", "runtime.status", "runtime.status"])
38
+ assert.equal(new Set(result.records.map((record) => record.writerEpoch)).size, 1)
39
+ assert.doesNotMatch(JSON.stringify(result.records), /must-not-survive/)
40
+
41
+ const filesBeforeClose = await readdir(directory)
42
+ assert(filesBeforeClose.some((name) => name.endsWith(".jsonl.zst")))
43
+ assert(filesBeforeClose.some((name) => name.endsWith(".jsonl.zst.idx")))
44
+ assert.equal(filesBeforeClose.some((name) => name.endsWith(".jsonl")), false)
45
+ assert.equal((await stat(directory)).mode & 0o777, 0o700)
46
+ for (const name of filesBeforeClose) assert.equal((await stat(join(directory, name))).mode & 0o777, 0o600)
47
+
48
+ const firstClose = logger.close()
49
+ const secondClose = logger.close()
50
+ assert.equal(firstClose, secondClose)
51
+ const finalStatus = await firstClose
52
+ assert.equal(finalStatus.state, "closed")
53
+ assert.equal(logger.log("process.started", {runtimeMode: "reverse-hub"}), "dropped")
54
+ assert((await readdir(directory)).some((name) => name.endsWith(".jsonl.zst.sealed")))
55
+ const closed = await readCompressedLogDirectory(directory)
56
+ assert(closed.records.some((record) => record.event === "log_batch_accepted"))
57
+ assert(closed.records.some((record) => record.event === "log_frame_committed"))
58
+ assert.equal(closed.records.at(-1)?.event, "log_closed")
59
+ })
60
+
61
+ test("compressed logger enforces one shared bounded queue with a critical reserve", async (t) => {
62
+ const root = await mkdtemp(join(tmpdir(), "socketduct-compressed-pressure-"))
63
+ const directory = join(root, "logs")
64
+ const logger = createCompressedJsonlFileLogger({
65
+ directory,
66
+ role: "reverse-client",
67
+ maxQueueBytes: 4096,
68
+ criticalReserveBytes: 1024,
69
+ frameMaxBytes: 1024,
70
+ flushIntervalMs: 60_000,
71
+ criticalFlushIntervalMs: 60_000
72
+ })
73
+ t.after(async () => { await logger.close().catch(() => {}); await rm(root, {recursive: true, force: true}) })
74
+
75
+ const outcomes = Array.from({length: 40}, () => logger.log("runtime.status", {virtualStreams: 1}))
76
+ assert(outcomes.includes("dropped") || outcomes.includes("coalesced"))
77
+ assert(logger.status().queuedBytes <= 4096)
78
+ assert.equal(logger.log("transport.disconnected", {
79
+ clientConnectionGeneration: 1,
80
+ disconnectIncidentId: "AAAAAAAAAAAAAAAAAAAAAA",
81
+ disconnectReason: "remote_eof",
82
+ incidentOrigin: "reverse_client",
83
+ physicalRole: "client",
84
+ fanOutCount: 1,
85
+ resumeSuccessCount: 0,
86
+ terminalStreamCount: 0
87
+ }), "accepted")
88
+ assert(logger.status().queuedBytes <= 4096)
89
+ await logger.flush()
90
+ })
91
+
92
+ test("compressed logger caps worker restarts and retains only bounded critical evidence after failure", async (t) => {
93
+ const root = await mkdtemp(join(tmpdir(), "socketduct-compressed-worker-death-"))
94
+ const directory = join(root, "logs")
95
+ const logger = createCompressedJsonlFileLogger({
96
+ directory,
97
+ role: "reverse-hub",
98
+ frameMaxBytes: 256,
99
+ maxSegmentBytes: 1024,
100
+ maxQueueBytes: 65_536,
101
+ criticalReserveBytes: 8192,
102
+ flushIntervalMs: 60_000,
103
+ criticalFlushIntervalMs: 60_000,
104
+ stderr: {write: () => true}
105
+ })
106
+ t.after(async () => { await logger.close().catch(() => {}); await rm(root, {recursive: true, force: true}) })
107
+ logger.log("process.started", {runtimeMode: "reverse-hub"})
108
+ await logger.flush()
109
+ await rm(directory, {recursive: true, force: true})
110
+
111
+ for (let index = 0; index < 20; index += 1) logger.log("process.started", {runtimeMode: "reverse-hub"})
112
+ await assert.rejects(() => logger.flush())
113
+ assert.equal(logger.status().state, "failed")
114
+ assert.equal(logger.status().workerRestarts, 3)
115
+ assert.equal(logger.log("runtime.status", {virtualStreams: 1}), "dropped")
116
+ assert.equal(logger.log("process.started", {runtimeMode: "reverse-hub"}), "accepted")
117
+ assert(logger.status().allocatedBytes <= 65_536)
118
+ })
119
+
120
+ test("compressed logger flush is a caller-time durability barrier", async (t) => {
121
+ const root = await mkdtemp(join(tmpdir(), "socketduct-compressed-barrier-"))
122
+ const logger = createCompressedJsonlFileLogger({
123
+ directory: join(root, "logs"),
124
+ role: "reverse-hub",
125
+ frameMaxBytes: 256,
126
+ flushIntervalMs: 60_000,
127
+ criticalFlushIntervalMs: 60_000,
128
+ maxQueueBytes: 1_000_000,
129
+ criticalReserveBytes: 1024
130
+ })
131
+ t.after(async () => { await logger.close().catch(() => {}); await rm(root, {recursive: true, force: true}) })
132
+ await logger.flush()
133
+ assert.equal(logger.log("process.started", {runtimeMode: "reverse-hub"}), "accepted")
134
+ const barrier = logger.flush()
135
+ for (let index = 0; index < 8; index += 1) assert.equal(logger.log("process.started", {runtimeMode: "reverse-hub"}), "accepted")
136
+ await barrier
137
+ const status = logger.status()
138
+ assert(BigInt(status.durableHighWater) >= 1n)
139
+ assert(BigInt(status.durableHighWater) < BigInt(status.acceptedHighWater))
140
+ })
141
+
142
+ test("compressed logger flushes immediately at frame size and critical deadlines do not slide", async (t) => {
143
+ const root = await mkdtemp(join(tmpdir(), "socketduct-compressed-deadline-"))
144
+ const directory = join(root, "logs")
145
+ const logger = createCompressedJsonlFileLogger({
146
+ directory,
147
+ role: "reverse-hub",
148
+ frameMaxBytes: 256,
149
+ flushIntervalMs: 60_000,
150
+ criticalFlushIntervalMs: 40,
151
+ maxQueueBytes: 1_000_000,
152
+ criticalReserveBytes: 1024
153
+ })
154
+ t.after(async () => { await logger.close().catch(() => {}); await rm(root, {recursive: true, force: true}) })
155
+ await logger.flush()
156
+
157
+ assert.equal(logger.log("runtime.status", {virtualStreams: 1}), "accepted")
158
+ assert.equal(logger.log("process.started", {runtimeMode: "reverse-hub"}), "accepted")
159
+ await waitUntil(() => BigInt(logger.status().durableHighWater) > 0n, 1_000)
160
+
161
+ await logger.flush()
162
+ const before = BigInt(logger.status().durableHighWater)
163
+ const producer = setInterval(() => logger.log("process.started", {runtimeMode: "reverse-hub"}), 5)
164
+ try {
165
+ await waitUntil(() => BigInt(logger.status().durableHighWater) > before, 500)
166
+ } finally {
167
+ clearInterval(producer)
168
+ }
169
+ const durable = await readCompressedLogDirectory(directory)
170
+ assert(durable.records.some((record) => record.event === "runtime.status"))
171
+ })
172
+
173
+ /**
174
+ * @param {() => boolean} predicate - Completion predicate.
175
+ * @param {number} timeoutMs - Deadline.
176
+ */
177
+ async function waitUntil(predicate, timeoutMs) {
178
+ const deadline = Date.now() + timeoutMs
179
+ while (!predicate()) {
180
+ if (Date.now() >= deadline) assert.fail("timed out waiting for compressed logger progress")
181
+ await new Promise((resolve) => setTimeout(resolve, 5))
182
+ }
183
+ }
@@ -0,0 +1,86 @@
1
+ /* eslint-disable jsdoc/escape-inline-tags, jsdoc/require-param-type, jsdoc/require-returns */
2
+ import assert from "node:assert/strict"
3
+ import {readlinkSync} from "node:fs"
4
+ import {mkdtemp, readFile, readdir, rm, utimes, writeFile} from "node:fs/promises"
5
+ import {tmpdir} from "node:os"
6
+ import {join} from "node:path"
7
+ import test from "node:test"
8
+
9
+ import {enforceLogRetention} from "../../../src/logging/log-retention.js"
10
+
11
+ test("retention removes only oldest sealed segments and preserves active evidence", async (t) => {
12
+ const directory = await mkdtemp(join(tmpdir(), "socketduct-retention-"))
13
+ t.after(() => rm(directory, {recursive: true, force: true}))
14
+ const old = await createSegment(directory, "reverse-hub-AAAAAAAAAAAAAAAAAAAAAA-123e4567-e89b-42d3-a456-426614174000", 700, true)
15
+ const newer = await createSegment(directory, "reverse-hub-BBBBBBBBBBBBBBBBBBBBBB-123e4567-e89b-42d3-a456-426614174001", 700, true)
16
+ const active = await createSegment(directory, "reverse-hub-CCCCCCCCCCCCCCCCCCCCCC-123e4567-e89b-42d3-a456-426614174002", 700, false)
17
+ await utimes(old.data, new Date(1_000), new Date(1_000))
18
+ await utimes(newer.data, new Date(2_000), new Date(2_000))
19
+ await utimes(active.data, new Date(3_000), new Date(3_000))
20
+
21
+ const result = await enforceLogRetention({directory, maxAgeMs: 60_000, maxBytes: 1800, now: () => new Date(3_000)})
22
+ const names = await readdir(directory)
23
+ assert.equal(names.some((name) => name.startsWith(old.name)), false)
24
+ assert(names.some((name) => name.startsWith(newer.name)))
25
+ assert(names.some((name) => name.startsWith(active.name)))
26
+ assert.equal(result.deletedSegments, 1)
27
+ })
28
+
29
+ test("retention fails closed when active or quarantined bytes prevent quota compliance", async (t) => {
30
+ const directory = await mkdtemp(join(tmpdir(), "socketduct-retention-blocked-"))
31
+ t.after(() => rm(directory, {recursive: true, force: true}))
32
+ const active = await createSegment(directory, "reverse-client-DDDDDDDDDDDDDDDDDDDDDD-123e4567-e89b-42d3-a456-426614174003", 1800, false)
33
+ await writeFile(`${active.data}.quarantine`, "LOG_EVIDENCE_INVALID", {mode: 0o600})
34
+
35
+ await assert.rejects(
36
+ () => enforceLogRetention({directory, maxAgeMs: 60_000, maxBytes: 1024, now: () => new Date(3_000)}),
37
+ (error) => error instanceof Error && /** @type {{code?: unknown}} */ (error).code === "LOG_RETENTION_QUOTA_BLOCKED"
38
+ )
39
+ assert((await readdir(directory)).some((name) => name.startsWith(active.name)))
40
+ })
41
+
42
+ test("retention reclaims stale ownership without displacing a live process instance", async (t) => {
43
+ const directory = await mkdtemp(join(tmpdir(), "socketduct-retention-owner-"))
44
+ t.after(() => rm(directory, {recursive: true, force: true}))
45
+ const lockPath = join(directory, ".socketduct-retention.lock")
46
+ const processStart = processStartIdentity(await readFile(`/proc/${process.pid}/stat`, "utf8"))
47
+ const tid = Number(readlinkSync("/proc/thread-self").split("/").at(-1))
48
+ const threadStart = processStartIdentity(await readFile(`/proc/${process.pid}/task/${tid}/stat`, "utf8"))
49
+ const staleTemporary = `.socketduct-retention-owner-${process.pid}-${BigInt(processStart) + 1n}-${tid}-${threadStart}-123e4567-e89b-42d3-a456-426614174099.tmp`
50
+ await writeFile(join(directory, staleTemporary), "incomplete", {mode: 0o600})
51
+
52
+ await writeFile(lockPath, `${JSON.stringify({version: 1, pid: process.pid, processStart: String(BigInt(processStart) + 1n), tid, threadStart})}\n`, {mode: 0o600})
53
+ assert.equal((await enforceLogRetention({directory, maxAgeMs: 60_000, maxBytes: 1024})).skipped, false)
54
+ assert.equal((await readdir(directory)).includes(staleTemporary), false)
55
+
56
+ await writeFile(lockPath, `${JSON.stringify({version: 1, pid: process.pid, processStart, tid: 2_147_483_647, threadStart: "1"})}\n`, {mode: 0o600})
57
+ assert.equal((await enforceLogRetention({directory, maxAgeMs: 60_000, maxBytes: 1024})).skipped, false)
58
+
59
+ await writeFile(lockPath, `${JSON.stringify({version: 1, pid: process.pid, processStart, tid, threadStart})}\n`, {mode: 0o600})
60
+ assert.equal((await enforceLogRetention({directory, maxAgeMs: 60_000, maxBytes: 1024})).skipped, true)
61
+ })
62
+
63
+ /**
64
+ * @param {string} directory - Fixture directory. @param {string} stem - Segment stem. @param {number} bytes - Data bytes. @param {boolean} sealed - Whether to seal.
65
+ * @param stem
66
+ * @param bytes
67
+ * @param sealed
68
+ */
69
+ async function createSegment(directory, stem, bytes, sealed) {
70
+ const name = `${stem}.jsonl.zst`
71
+ const data = join(directory, name)
72
+ await writeFile(data, Buffer.alloc(bytes), {mode: 0o600})
73
+ await writeFile(`${data}.idx`, Buffer.alloc(48), {mode: 0o600})
74
+ if (sealed) await writeFile(`${data}.sealed`, "", {mode: 0o600})
75
+ return {name, data}
76
+ }
77
+
78
+ /** @param {string} value - Linux proc stat contents. @returns {string} Kernel process start ticks. */
79
+ function processStartIdentity(value) {
80
+ const close = value.lastIndexOf(")")
81
+ const fields = value.slice(close + 2).trim().split(/\s+/)
82
+ const start = fields[19]
83
+ if (start === undefined) assert.fail("process stat omits start time")
84
+ assert.match(start, /^[0-9]+$/)
85
+ return start
86
+ }
@@ -0,0 +1,104 @@
1
+ import assert from "node:assert/strict"
2
+ import {createHash, randomBytes} from "node:crypto"
3
+ import test from "node:test"
4
+
5
+ import {
6
+ INDEX_ENTRY_BYTES,
7
+ INDEX_HEADER_BYTES,
8
+ createIndexEntry,
9
+ createIndexHeader,
10
+ parseIndexEntry,
11
+ parseIndexHeader,
12
+ uuidBytes
13
+ } from "../../../src/logging/log-segment-index.js"
14
+
15
+ test("index v1 header and entry round-trip in the exact fixed-width format", () => {
16
+ const segmentUuid = uuidBytes("123e4567-e89b-42d3-a456-426614174000")
17
+ const writerEpoch = randomBytes(16)
18
+ const batchId = randomBytes(16)
19
+ const frameHash = createHash("sha256").update("frame").digest()
20
+ const header = createIndexHeader({segmentUuid, writerEpoch})
21
+ const entry = createIndexEntry({
22
+ segmentUuid,
23
+ writerEpoch,
24
+ batchId,
25
+ firstSequence: 1n,
26
+ lastSequence: 3n,
27
+ dataOffset: 0n,
28
+ compressedLength: 91n,
29
+ uncompressedLength: 240n,
30
+ recordCount: 3,
31
+ minTimestamp: 1_795_000_000_000n,
32
+ maxTimestamp: 1_795_000_000_010n,
33
+ frameSha256: frameHash
34
+ })
35
+
36
+ assert.equal(header.length, INDEX_HEADER_BYTES)
37
+ assert.equal(entry.length, INDEX_ENTRY_BYTES)
38
+ assert.deepEqual(parseIndexHeader(header), {version: 1, segmentUuid, writerEpoch})
39
+ assert.deepEqual(parseIndexEntry(entry, {segmentUuid, writerEpoch}), {
40
+ version: 1,
41
+ flags: 0,
42
+ segmentUuid,
43
+ writerEpoch,
44
+ batchId,
45
+ firstSequence: 1n,
46
+ lastSequence: 3n,
47
+ dataOffset: 0n,
48
+ compressedLength: 91n,
49
+ uncompressedLength: 240n,
50
+ recordCount: 3,
51
+ minTimestamp: 1_795_000_000_000n,
52
+ maxTimestamp: 1_795_000_000_010n,
53
+ frameSha256: frameHash
54
+ })
55
+ })
56
+
57
+ test("index CRC and identity corruption fail closed", () => {
58
+ const segmentUuid = randomBytes(16)
59
+ const writerEpoch = randomBytes(16)
60
+ const header = createIndexHeader({segmentUuid, writerEpoch})
61
+ header[20] = /** @type {number} */ (header[20]) ^ 0xff
62
+ assert.throws(() => parseIndexHeader(header), /CRC32C/)
63
+
64
+ const entry = createIndexEntry({
65
+ segmentUuid,
66
+ writerEpoch,
67
+ batchId: randomBytes(16),
68
+ firstSequence: 8n,
69
+ lastSequence: 8n,
70
+ dataOffset: 10n,
71
+ compressedLength: 20n,
72
+ uncompressedLength: 30n,
73
+ recordCount: 1,
74
+ minTimestamp: 4n,
75
+ maxTimestamp: 4n,
76
+ frameSha256: randomBytes(32)
77
+ })
78
+ entry[151] = /** @type {number} */ (entry[151]) ^ 0xff
79
+ assert.throws(() => parseIndexEntry(entry, {segmentUuid, writerEpoch}), /CRC32C/)
80
+ })
81
+
82
+ test("index bytes contain no searchable diagnostic dimensions", () => {
83
+ const forbidden = ["minisforum-zyxel", "disconnect-incident", "GATEWAY_DISCONNECTED", "docker.internal"]
84
+ const segmentUuid = randomBytes(16)
85
+ const writerEpoch = randomBytes(16)
86
+ const encoded = Buffer.concat([
87
+ createIndexHeader({segmentUuid, writerEpoch}),
88
+ createIndexEntry({
89
+ segmentUuid,
90
+ writerEpoch,
91
+ batchId: randomBytes(16),
92
+ firstSequence: 1n,
93
+ lastSequence: 1n,
94
+ dataOffset: 0n,
95
+ compressedLength: 1n,
96
+ uncompressedLength: 1n,
97
+ recordCount: 1,
98
+ minTimestamp: 1n,
99
+ maxTimestamp: 1n,
100
+ frameSha256: randomBytes(32)
101
+ })
102
+ ])
103
+ for (const value of forbidden) assert.equal(encoded.includes(Buffer.from(value)), false)
104
+ })
@@ -3,7 +3,7 @@ import {chmod, mkdtemp, mkdir, readFile, readdir, rename, rm, stat, utimes, writ
3
3
  import {tmpdir} from "node:os"
4
4
  import {join} from "node:path"
5
5
 
6
- import {createJsonlFileLogger, createStructuredLogger, errorDiagnostics, logDiagnostic} from "../../../src/logging/structured-logger.js"
6
+ import {createJsonlFileLogger, createStructuredLogger, errorDiagnostics, formatDiagnosticLogLine, logDiagnostic} from "../../../src/logging/structured-logger.js"
7
7
 
8
8
  const FIXED_TIME = "2026-09-05T12:34:56.789Z"
9
9
 
@@ -21,6 +21,19 @@ function parseLines(contents) {
21
21
  }
22
22
 
23
23
  describe("structured logger", {timeoutMs: 15000}, () => {
24
+ it("accepts only the exact producer-owned compressed envelope", () => {
25
+ expect(() => formatDiagnosticLogLine({
26
+ role: "reverse-hub",
27
+ event: "process.started",
28
+ envelope: /** @type {{writerEpoch: string, eventSequence: string}} */ ({writerEpoch: "AAAAAAAAAAAAAAAAAAAAAA", eventSequence: "1", role: "peer-role"})
29
+ })).toThrow(/envelope/)
30
+ expect(() => formatDiagnosticLogLine({
31
+ role: "reverse-hub",
32
+ event: "process.started",
33
+ envelope: {writerEpoch: "not-canonical", eventSequence: "1"}
34
+ })).toThrow(/writerEpoch/)
35
+ })
36
+
24
37
  it("structured logger writes one allowlisted bounded JSON object per line", () => {
25
38
  /** @type {string[]} */
26
39
  const lines = []
@@ -0,0 +1,24 @@
1
+ import assert from "node:assert/strict"
2
+ import test from "node:test"
3
+
4
+ import {compressLogFrame, decompressLogFrame} from "../../../src/logging/zstd-frame-codec.js"
5
+
6
+ test("log codec emits independently decodable standard zstd frames with checksums at level 19", async () => {
7
+ const firstRaw = Buffer.from('{"event":"first"}\n')
8
+ const secondRaw = Buffer.from('{"event":"second"}\n')
9
+ const first = await compressLogFrame(firstRaw)
10
+ const second = await compressLogFrame(secondRaw)
11
+
12
+ assert.deepEqual(first.subarray(0, 4), Buffer.from([0x28, 0xb5, 0x2f, 0xfd]))
13
+ assert.notEqual(/** @type {number} */ (first[4]) & 0x04, 0, "zstd frame checksum flag must be set")
14
+ assert.deepEqual(await decompressLogFrame(first), firstRaw)
15
+ assert.deepEqual(await decompressLogFrame(second), secondRaw)
16
+ assert.notDeepEqual(first, second)
17
+ })
18
+
19
+ test("log codec never falls back to plaintext", async () => {
20
+ const raw = Buffer.from("small\n")
21
+ const frame = await compressLogFrame(raw)
22
+ assert.equal(frame.equals(raw), false)
23
+ await assert.rejects(() => decompressLogFrame(Buffer.from("small\n")))
24
+ })