socketduct 0.1.24 → 0.1.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +2 -2
- package/changelog.d/20261010130509-bound-log-frame-compression-memory.md +1 -0
- package/docs/runbooks/jsonl-integrity-rollout.md +8 -6
- package/package.json +2 -2
- package/scripts/test.js +9 -14
- package/src/client/multiplexed-transport-set.js +20 -2
- package/src/logging/compressed-log-writer.js +1 -1
- package/src/logging/zstd-frame-codec.js +15 -1
- package/src/reverse/client.js +44 -1
- package/src/reverse/hub.js +10 -1
- package/src/reverse/runtime.js +11 -2
- package/test/e2e/gateway-process.test.js +93 -92
- package/test/e2e/logger-shutdown-process.test.js +40 -37
- package/test/e2e/reverse-client-recovery-process.test.js +107 -103
- package/test/e2e/reverse-gateway-process.test.js +328 -318
- package/test/integration/initial-target-recovery.test.js +333 -334
- package/test/integration/logging-cli.test.js +121 -119
- package/test/integration/logging-compressed-recovery.test.js +96 -95
- package/test/integration/multiplexed-compatibility.test.js +178 -177
- package/test/integration/multiplexed-corrections.test.js +360 -359
- package/test/integration/multiplexed-failures.test.js +253 -251
- package/test/integration/multiplexed-transport-set.test.js +334 -322
- package/test/integration/multiplexed-transport.test.js +135 -134
- package/test/integration/reverse-gateway-stream-id-reset.test.js +90 -89
- package/test/integration/reverse-gateway.test.js +2309 -2218
- package/test/integration/reverse-resume-continuity.test.js +219 -218
- package/test/integration/reverse-route-class.test.js +363 -278
- package/test/integration/socketduct-http-agent.test.js +314 -312
- package/test/integration/socketduct-https-agent.test.js +516 -485
- package/test/integration/stream-compression.test.js +862 -824
- package/test/integration/tcp-forwarding.test.js +2550 -2447
- package/test/integration/tls-forwarding.test.js +593 -592
- package/test/unit/auth/auth.test.js +43 -44
- package/test/unit/auth/static-token-store.test.js +9 -8
- package/test/unit/client/connect.test.js +41 -40
- package/test/unit/client/deferred-virtual-socket.test.js +95 -94
- package/test/unit/client/multiplexed-spool-budget.test.js +155 -154
- package/test/unit/client/multiplexed-transport-set.test.js +845 -732
- package/test/unit/client/socketduct-socket.test.js +768 -771
- package/test/unit/client/virtual-socket.test.js +198 -168
- package/test/unit/config/docker-gateway-files.test.js +224 -224
- package/test/unit/config/environment.test.js +472 -492
- package/test/unit/index.test.js +53 -52
- package/test/unit/logging/compressed-log-worker.test.js +23 -22
- package/test/unit/logging/compressed-log-writer.test.js +155 -154
- package/test/unit/logging/log-retention.test.js +52 -47
- package/test/unit/logging/log-segment-index.test.js +84 -81
- package/test/unit/logging/zstd-frame-codec.test.js +28 -19
- package/test/unit/protocol/compression.test.js +254 -298
- package/test/unit/protocol/frame.test.js +280 -276
- package/test/unit/protocol/v3-compression.test.js +21 -23
- package/test/unit/protocol/v3-frame.test.js +42 -41
- package/test/unit/release-configuration.test.js +15 -12
- package/test/unit/reverse/disconnect-attribution.test.js +65 -64
- package/test/unit/reverse-terminal-outcomes.test.js +53 -50
- package/test/unit/server/control-handler.test.js +232 -231
- package/test/unit/server/primary.test.js +51 -49
- package/test/unit/server/relay-runtime.test.js +280 -294
- package/test/unit/sessions/session.test.js +48 -62
- package/test/unit/spool/spool-file.test.js +401 -385
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
import
|
|
1
|
+
import {describe, expect, it} from "@velocious/testing"
|
|
2
2
|
import {randomBytes} from "node:crypto"
|
|
3
|
-
import test from "node:test"
|
|
4
3
|
import zlib from "node:zlib"
|
|
5
4
|
|
|
6
5
|
import {
|
|
@@ -44,378 +43,335 @@ function incompressibleChunk(size) {
|
|
|
44
43
|
return randomBytes(size)
|
|
45
44
|
}
|
|
46
45
|
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
46
|
+
describe("stream payload compression", () => {
|
|
47
|
+
it("availableCodecs advertises zstd only when the runtime supports it", () => {
|
|
48
|
+
expect(availableCodecs()).toEqual(zstdAvailable() ? [ZSTD_CODEC] : [])
|
|
49
|
+
})
|
|
50
50
|
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
})
|
|
51
|
+
it("negotiateCodec selects the first mutually supported codec", () => {
|
|
52
|
+
expect(negotiateCodec(["zstd"], ["zstd"])).toBe("zstd")
|
|
53
|
+
expect(negotiateCodec(["brotli", "zstd"], ["zstd"])).toBe("zstd")
|
|
54
|
+
})
|
|
55
55
|
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
})
|
|
56
|
+
it("negotiateCodec returns undefined for absent, empty, unknown or malformed offers", () => {
|
|
57
|
+
expect(negotiateCodec(undefined, ["zstd"])).toBeUndefined()
|
|
58
|
+
expect(negotiateCodec([], ["zstd"])).toBeUndefined()
|
|
59
|
+
expect(negotiateCodec(["brotli"], ["zstd"])).toBeUndefined()
|
|
60
|
+
expect(negotiateCodec("zstd", ["zstd"])).toBeUndefined()
|
|
61
|
+
expect(negotiateCodec({codec: "zstd"}, ["zstd"])).toBeUndefined()
|
|
62
|
+
expect(negotiateCodec(["zstd"], [])).toBeUndefined()
|
|
63
|
+
})
|
|
64
64
|
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
65
|
+
it("encodeStreamPayload round-trips a compressible payload and shrinks it", {skip: !zstdAvailable()}, async () => {
|
|
66
|
+
const chunk = compressibleChunk(8192)
|
|
67
|
+
const encoded = await encodeStreamPayload(chunk, ZSTD_CODEC)
|
|
68
68
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
69
|
+
expect(encoded.encoding).toBe(ZSTD_CODEC)
|
|
70
|
+
expect(encoded.dataLength).toBe(chunk.length)
|
|
71
|
+
expect(encoded.payload.length < chunk.length).toBeTruthy()
|
|
72
72
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
})
|
|
73
|
+
const decoded = await decodeStreamPayload(
|
|
74
|
+
{type: "stream.data", encoding: encoded.encoding, dataLength: encoded.dataLength},
|
|
75
|
+
encoded.payload,
|
|
76
|
+
ZSTD_CODEC
|
|
77
|
+
)
|
|
78
|
+
expect(Buffer.from(decoded).equals(Buffer.from(chunk))).toBe(true)
|
|
79
|
+
})
|
|
80
80
|
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
81
|
+
it("encodeStreamPayload sends raw when no codec is negotiated", async () => {
|
|
82
|
+
const chunk = compressibleChunk(8192)
|
|
83
|
+
const encoded = await encodeStreamPayload(chunk, undefined)
|
|
84
84
|
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
})
|
|
85
|
+
expect(encoded.encoding).toBeUndefined()
|
|
86
|
+
expect(encoded.dataLength).toBeUndefined()
|
|
87
|
+
expect(Buffer.from(encoded.payload).equals(Buffer.from(chunk))).toBe(true)
|
|
88
|
+
})
|
|
89
89
|
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
90
|
+
it("encodeStreamPayload sends payloads below the minimum size raw", {skip: !zstdAvailable()}, async () => {
|
|
91
|
+
const chunk = compressibleChunk(64)
|
|
92
|
+
const encoded = await encodeStreamPayload(chunk, ZSTD_CODEC)
|
|
93
93
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
})
|
|
94
|
+
expect(encoded.encoding).toBeUndefined()
|
|
95
|
+
expect(Buffer.from(encoded.payload).toString("hex")).toBe(Buffer.from(chunk).toString("hex"))
|
|
96
|
+
})
|
|
97
97
|
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
98
|
+
it("encodeStreamPayload never grows a frame for incompressible data", {skip: !zstdAvailable()}, async () => {
|
|
99
|
+
const chunk = incompressibleChunk(16384)
|
|
100
|
+
const encoded = await encodeStreamPayload(chunk, ZSTD_CODEC)
|
|
101
101
|
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
})
|
|
102
|
+
expect(encoded.encoding).toBeUndefined()
|
|
103
|
+
expect(encoded.payload.length <= chunk.length).toBeTruthy()
|
|
104
|
+
expect(Buffer.from(encoded.payload).equals(Buffer.from(chunk))).toBe(true)
|
|
105
|
+
})
|
|
106
106
|
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
})
|
|
107
|
+
it("decodeStreamPayload passes raw payloads through untouched", async () => {
|
|
108
|
+
const chunk = compressibleChunk(1024)
|
|
109
|
+
expect(Buffer.from(await decodeStreamPayload({type: "stream.data"}, chunk, ZSTD_CODEC)).equals(Buffer.from(chunk))).toBe(true)
|
|
110
|
+
expect(Buffer.from(await decodeStreamPayload({type: "stream.data"}, chunk, undefined)).equals(Buffer.from(chunk))).toBe(true)
|
|
111
|
+
})
|
|
112
112
|
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
await assert.rejects(
|
|
117
|
-
() => decodeStreamPayload({type: "stream.data"}, chunk, undefined, {maxDecompressedBytes: 1024}),
|
|
118
|
-
CompressionProtocolError
|
|
119
|
-
)
|
|
120
|
-
await assert.rejects(
|
|
121
|
-
() => decodeStreamPayload({type: "stream.data"}, chunk, ZSTD_CODEC, {maxDecompressedBytes: 1024}),
|
|
122
|
-
CompressionProtocolError
|
|
123
|
-
)
|
|
124
|
-
})
|
|
113
|
+
it("decodeStreamPayload enforces the uncompressed maximum on raw payloads too", async () => {
|
|
114
|
+
const chunk = compressibleChunk(4096)
|
|
125
115
|
|
|
126
|
-
|
|
127
|
-
|
|
116
|
+
await expect(decodeStreamPayload({type: "stream.data"}, chunk, undefined, {maxDecompressedBytes: 1024})).rejects.toThrow(CompressionProtocolError)
|
|
117
|
+
await expect(decodeStreamPayload({type: "stream.data"}, chunk, ZSTD_CODEC, {maxDecompressedBytes: 1024})).rejects.toThrow(CompressionProtocolError)
|
|
118
|
+
})
|
|
128
119
|
|
|
129
|
-
|
|
130
|
-
|
|
120
|
+
it("decodeStreamPayload still passes a raw payload within the uncompressed maximum", async () => {
|
|
121
|
+
const chunk = compressibleChunk(1024)
|
|
131
122
|
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
() => decodeStreamPayload({type: "stream.data", dataLength: 10}, Buffer.alloc(4), ZSTD_CODEC),
|
|
135
|
-
CompressionProtocolError
|
|
136
|
-
)
|
|
137
|
-
})
|
|
123
|
+
expect(Buffer.from(await decodeStreamPayload({type: "stream.data"}, chunk, undefined, {maxDecompressedBytes: 1024})).equals(Buffer.from(chunk))).toBe(true)
|
|
124
|
+
})
|
|
138
125
|
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
126
|
+
it("decodeStreamPayload rejects dataLength without an encoding", async () => {
|
|
127
|
+
await expect(decodeStreamPayload({type: "stream.data", dataLength: 10}, Buffer.alloc(4), ZSTD_CODEC)).rejects.toThrow(CompressionProtocolError)
|
|
128
|
+
})
|
|
142
129
|
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
)
|
|
147
|
-
})
|
|
130
|
+
it("decodeStreamPayload rejects an encoding that was never negotiated", {skip: !zstdAvailable()}, async () => {
|
|
131
|
+
const chunk = compressibleChunk(4096)
|
|
132
|
+
const encoded = await encodeStreamPayload(chunk, ZSTD_CODEC)
|
|
148
133
|
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
CompressionProtocolError
|
|
157
|
-
)
|
|
158
|
-
})
|
|
134
|
+
await expect(decodeStreamPayload({type: "stream.data", encoding: "zstd", dataLength: chunk.length}, encoded.payload, undefined)).rejects.toThrow(CompressionProtocolError)
|
|
135
|
+
})
|
|
136
|
+
|
|
137
|
+
it("decodeStreamPayload rejects an unknown or non-string encoding", async () => {
|
|
138
|
+
await expect(decodeStreamPayload({type: "stream.data", encoding: "brotli", dataLength: 32}, Buffer.alloc(8), ZSTD_CODEC)).rejects.toThrow(CompressionProtocolError)
|
|
139
|
+
await expect(decodeStreamPayload({type: "stream.data", encoding: 7, dataLength: 32}, Buffer.alloc(8), ZSTD_CODEC)).rejects.toThrow(CompressionProtocolError)
|
|
140
|
+
})
|
|
159
141
|
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
142
|
+
it("decodeStreamPayload rejects malformed dataLength values", {skip: !zstdAvailable()}, async () => {
|
|
143
|
+
const chunk = compressibleChunk(4096)
|
|
144
|
+
const encoded = await encodeStreamPayload(chunk, ZSTD_CODEC)
|
|
163
145
|
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
)
|
|
169
|
-
}
|
|
170
|
-
})
|
|
146
|
+
for (const dataLength of [undefined, 0, -1, 1.5, "4096", Number.MAX_SAFE_INTEGER + 2]) {
|
|
147
|
+
await expect(decodeStreamPayload({type: "stream.data", encoding: ZSTD_CODEC, dataLength}, encoded.payload, ZSTD_CODEC)).rejects.toThrow(CompressionProtocolError)
|
|
148
|
+
}
|
|
149
|
+
})
|
|
171
150
|
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
151
|
+
it("decodeStreamPayload rejects a dataLength above the configured maximum", {skip: !zstdAvailable()}, async () => {
|
|
152
|
+
const chunk = compressibleChunk(4096)
|
|
153
|
+
const encoded = await encodeStreamPayload(chunk, ZSTD_CODEC)
|
|
175
154
|
|
|
176
|
-
|
|
177
|
-
() => decodeStreamPayload(
|
|
155
|
+
await expect(decodeStreamPayload(
|
|
178
156
|
{type: "stream.data", encoding: ZSTD_CODEC, dataLength: chunk.length},
|
|
179
157
|
encoded.payload,
|
|
180
158
|
ZSTD_CODEC,
|
|
181
159
|
{maxDecompressedBytes: 1024}
|
|
182
|
-
)
|
|
183
|
-
|
|
184
|
-
)
|
|
185
|
-
})
|
|
160
|
+
)).rejects.toThrow(CompressionProtocolError)
|
|
161
|
+
})
|
|
186
162
|
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
163
|
+
it("decodeStreamPayload rejects a compressed payload above the configured maximum", {skip: !zstdAvailable()}, async () => {
|
|
164
|
+
const chunk = compressibleChunk(4096)
|
|
165
|
+
const encoded = await encodeStreamPayload(chunk, ZSTD_CODEC)
|
|
190
166
|
|
|
191
|
-
|
|
192
|
-
() => decodeStreamPayload(
|
|
167
|
+
await expect(decodeStreamPayload(
|
|
193
168
|
{type: "stream.data", encoding: ZSTD_CODEC, dataLength: chunk.length},
|
|
194
169
|
encoded.payload,
|
|
195
170
|
ZSTD_CODEC,
|
|
196
171
|
{maxCompressedBytes: 8}
|
|
197
|
-
)
|
|
198
|
-
|
|
199
|
-
)
|
|
200
|
-
})
|
|
172
|
+
)).rejects.toThrow(CompressionProtocolError)
|
|
173
|
+
})
|
|
201
174
|
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
CompressionProtocolError
|
|
206
|
-
)
|
|
207
|
-
})
|
|
175
|
+
it("decodeStreamPayload rejects an empty compressed payload", async () => {
|
|
176
|
+
await expect(decodeStreamPayload({type: "stream.data", encoding: ZSTD_CODEC, dataLength: 16}, Buffer.alloc(0), ZSTD_CODEC)).rejects.toThrow(CompressionProtocolError)
|
|
177
|
+
})
|
|
208
178
|
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
179
|
+
it("decodeStreamPayload rejects payloads whose expansion ratio is too high", {skip: !zstdAvailable()}, async () => {
|
|
180
|
+
const bomb = Buffer.alloc(4 * 1024 * 1024, 0)
|
|
181
|
+
const compressed = await zstdCompress(bomb)
|
|
212
182
|
|
|
213
|
-
|
|
214
|
-
() => decodeStreamPayload(
|
|
183
|
+
await expect(decodeStreamPayload(
|
|
215
184
|
{type: "stream.data", encoding: ZSTD_CODEC, dataLength: bomb.length},
|
|
216
185
|
compressed,
|
|
217
186
|
ZSTD_CODEC
|
|
218
|
-
)
|
|
219
|
-
|
|
220
|
-
)
|
|
221
|
-
})
|
|
187
|
+
)).rejects.toThrow(CompressionProtocolError)
|
|
188
|
+
})
|
|
222
189
|
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
190
|
+
it("decodeStreamPayload refuses to expand a bomb beyond its advertised dataLength", {skip: !zstdAvailable()}, async () => {
|
|
191
|
+
const bomb = Buffer.alloc(4 * 1024 * 1024, 0)
|
|
192
|
+
const compressed = await zstdCompress(bomb)
|
|
226
193
|
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
() => decodeStreamPayload(
|
|
194
|
+
// Understates the true size: decompression must be bounded, not allocate 4 MiB.
|
|
195
|
+
await expect(decodeStreamPayload(
|
|
230
196
|
{type: "stream.data", encoding: ZSTD_CODEC, dataLength: 1024},
|
|
231
197
|
compressed,
|
|
232
198
|
ZSTD_CODEC,
|
|
233
199
|
{maxExpansionRatio: Number.MAX_SAFE_INTEGER}
|
|
234
|
-
)
|
|
235
|
-
|
|
236
|
-
)
|
|
237
|
-
})
|
|
200
|
+
)).rejects.toThrow(CompressionProtocolError)
|
|
201
|
+
})
|
|
238
202
|
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
203
|
+
it("decodeStreamPayload rejects a dataLength larger than the true decompressed size", {skip: !zstdAvailable()}, async () => {
|
|
204
|
+
const chunk = compressibleChunk(4096)
|
|
205
|
+
const encoded = await encodeStreamPayload(chunk, ZSTD_CODEC)
|
|
242
206
|
|
|
243
|
-
|
|
244
|
-
() => decodeStreamPayload(
|
|
207
|
+
await expect(decodeStreamPayload(
|
|
245
208
|
{type: "stream.data", encoding: ZSTD_CODEC, dataLength: chunk.length + 1},
|
|
246
209
|
encoded.payload,
|
|
247
210
|
ZSTD_CODEC
|
|
248
|
-
)
|
|
249
|
-
|
|
250
|
-
)
|
|
251
|
-
})
|
|
211
|
+
)).rejects.toThrow(CompressionProtocolError)
|
|
212
|
+
})
|
|
252
213
|
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
214
|
+
it("decodeStreamPayload rejects a corrupt compressed payload", {skip: !zstdAvailable()}, async () => {
|
|
215
|
+
const chunk = compressibleChunk(4096)
|
|
216
|
+
const encoded = await encodeStreamPayload(chunk, ZSTD_CODEC)
|
|
217
|
+
const corrupted = Buffer.from(encoded.payload)
|
|
218
|
+
corrupted.writeUInt8(corrupted.readUInt8(corrupted.length - 1) ^ 0xff, corrupted.length - 1)
|
|
219
|
+
corrupted.writeUInt8(corrupted.readUInt8(10) ^ 0xff, 10)
|
|
259
220
|
|
|
260
|
-
|
|
261
|
-
() => decodeStreamPayload(
|
|
221
|
+
await expect(decodeStreamPayload(
|
|
262
222
|
{type: "stream.data", encoding: ZSTD_CODEC, dataLength: chunk.length},
|
|
263
223
|
corrupted,
|
|
264
224
|
ZSTD_CODEC
|
|
265
|
-
)
|
|
266
|
-
|
|
267
|
-
)
|
|
268
|
-
})
|
|
225
|
+
)).rejects.toThrow(CompressionProtocolError)
|
|
226
|
+
})
|
|
269
227
|
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
228
|
+
it("decodeStreamPayload rejects a truncated compressed payload", {skip: !zstdAvailable()}, async () => {
|
|
229
|
+
const chunk = compressibleChunk(4096)
|
|
230
|
+
const encoded = await encodeStreamPayload(chunk, ZSTD_CODEC)
|
|
273
231
|
|
|
274
|
-
|
|
275
|
-
() => decodeStreamPayload(
|
|
232
|
+
await expect(decodeStreamPayload(
|
|
276
233
|
{type: "stream.data", encoding: ZSTD_CODEC, dataLength: chunk.length},
|
|
277
234
|
encoded.payload.subarray(0, encoded.payload.length - 4),
|
|
278
235
|
ZSTD_CODEC
|
|
279
|
-
)
|
|
280
|
-
|
|
281
|
-
)
|
|
282
|
-
})
|
|
236
|
+
)).rejects.toThrow(CompressionProtocolError)
|
|
237
|
+
})
|
|
283
238
|
|
|
284
|
-
|
|
285
|
-
|
|
239
|
+
it("AdaptiveCompressor backs off on incompressible streams and re-probes", () => {
|
|
240
|
+
const adaptive = new AdaptiveCompressor()
|
|
286
241
|
|
|
287
|
-
|
|
288
|
-
|
|
242
|
+
expect(adaptive.shouldCompress()).toBe(true)
|
|
243
|
+
adaptive.record(1.0)
|
|
289
244
|
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
})
|
|
245
|
+
// Backs off for at least one chunk after an incompressible probe.
|
|
246
|
+
expect(adaptive.shouldCompress()).toBe(false)
|
|
247
|
+
// Then re-probes.
|
|
248
|
+
expect(adaptive.shouldCompress()).toBe(true)
|
|
249
|
+
})
|
|
295
250
|
|
|
296
|
-
|
|
297
|
-
|
|
251
|
+
it("AdaptiveCompressor keeps compressing while the stream compresses well", () => {
|
|
252
|
+
const adaptive = new AdaptiveCompressor()
|
|
298
253
|
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
})
|
|
254
|
+
for (let index = 0; index < 5; index += 1) {
|
|
255
|
+
expect(adaptive.shouldCompress()).toBe(true)
|
|
256
|
+
adaptive.record(0.3)
|
|
257
|
+
}
|
|
258
|
+
})
|
|
304
259
|
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
260
|
+
it("AdaptiveCompressor widens the skip window as incompressible chunks continue", () => {
|
|
261
|
+
const adaptive = new AdaptiveCompressor()
|
|
262
|
+
let skipped = 0
|
|
308
263
|
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
264
|
+
for (let round = 0; round < 4; round += 1) {
|
|
265
|
+
while (!adaptive.shouldCompress()) {
|
|
266
|
+
skipped += 1
|
|
267
|
+
}
|
|
268
|
+
adaptive.record(1.0)
|
|
312
269
|
}
|
|
313
|
-
adaptive.record(1.0)
|
|
314
|
-
}
|
|
315
270
|
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
})
|
|
271
|
+
// 1 + 2 + 4 skipped chunks across the three back-off rounds that follow a probe.
|
|
272
|
+
expect(skipped).toBe(7)
|
|
273
|
+
})
|
|
319
274
|
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
})
|
|
275
|
+
it("encodeStreamPayload skips compression while the adaptive window is closed", {skip: !zstdAvailable()}, async () => {
|
|
276
|
+
const adaptive = new AdaptiveCompressor()
|
|
277
|
+
const incompressible = incompressibleChunk(8192)
|
|
278
|
+
const compressible = compressibleChunk(8192)
|
|
279
|
+
|
|
280
|
+
// First probe is incompressible, which closes the window.
|
|
281
|
+
expect((await encodeStreamPayload(incompressible, ZSTD_CODEC, adaptive)).encoding).toBeUndefined()
|
|
282
|
+
// Next chunk is skipped without spending CPU, even though it would compress well.
|
|
283
|
+
expect((await encodeStreamPayload(compressible, ZSTD_CODEC, adaptive)).encoding).toBeUndefined()
|
|
284
|
+
// The re-probe compresses it.
|
|
285
|
+
expect((await encodeStreamPayload(compressible, ZSTD_CODEC, adaptive)).encoding).toBe(ZSTD_CODEC)
|
|
286
|
+
})
|
|
332
287
|
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
})
|
|
288
|
+
it("encodeStreamPayload yields to the event loop and does not block synchronously", {skip: !zstdAvailable()}, async () => {
|
|
289
|
+
const chunk = compressibleChunk(16384)
|
|
290
|
+
|
|
291
|
+
// Two concurrent encode calls must both complete with correct results.
|
|
292
|
+
// The second call must not be blocked by the first.
|
|
293
|
+
const [encoded1, encoded2] = await Promise.all([
|
|
294
|
+
encodeStreamPayload(chunk, ZSTD_CODEC),
|
|
295
|
+
encodeStreamPayload(chunk, ZSTD_CODEC)
|
|
296
|
+
])
|
|
297
|
+
|
|
298
|
+
expect(encoded1.encoding).toBe(ZSTD_CODEC)
|
|
299
|
+
expect(encoded1.payload.length < chunk.length).toBeTruthy()
|
|
300
|
+
expect(encoded2.encoding).toBe(ZSTD_CODEC)
|
|
301
|
+
expect(encoded2.payload.length < chunk.length).toBeTruthy()
|
|
302
|
+
// Independent frames produce independent compressed representations.
|
|
303
|
+
expect(Buffer.concat([encoded1.payload, encoded2.payload]).length < chunk.length * 2).toBe(true)
|
|
304
|
+
})
|
|
350
305
|
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
306
|
+
it("decodeStreamPayload yields to the event loop and does not block synchronously", {skip: !zstdAvailable()}, async () => {
|
|
307
|
+
const chunk = compressibleChunk(8192)
|
|
308
|
+
const encoded = await encodeStreamPayload(chunk, ZSTD_CODEC)
|
|
309
|
+
const header = {type: "stream.data", encoding: ZSTD_CODEC, dataLength: chunk.length}
|
|
355
310
|
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
311
|
+
// Two concurrent decode calls must both complete without blocking each other.
|
|
312
|
+
const [decoded1, decoded2] = await Promise.all([
|
|
313
|
+
decodeStreamPayload(header, encoded.payload, ZSTD_CODEC),
|
|
314
|
+
decodeStreamPayload(header, encoded.payload, ZSTD_CODEC)
|
|
315
|
+
])
|
|
361
316
|
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
})
|
|
317
|
+
expect(Buffer.from(decoded1).equals(Buffer.from(chunk))).toBe(true)
|
|
318
|
+
expect(Buffer.from(decoded2).equals(Buffer.from(chunk))).toBe(true)
|
|
319
|
+
})
|
|
365
320
|
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
321
|
+
it("async zstd encoding preserves frame ordering when serialized", {skip: !zstdAvailable()}, async () => {
|
|
322
|
+
const chunks = [
|
|
323
|
+
compressibleChunk(4096),
|
|
324
|
+
compressibleChunk(8192),
|
|
325
|
+
compressibleChunk(16384)
|
|
326
|
+
]
|
|
327
|
+
|
|
328
|
+
// Sequential encoding: each completes fully before the next starts.
|
|
329
|
+
// The test proves the async API returns in order when awaited in sequence.
|
|
330
|
+
const frames = []
|
|
331
|
+
for (const chunk of chunks) {
|
|
332
|
+
frames.push({chunk, encoded: await encodeStreamPayload(chunk, ZSTD_CODEC)})
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
expect(frames.length).toBe(3)
|
|
336
|
+
for (const frame of frames) {
|
|
337
|
+
expect(frame.encoded.encoding).toBe(ZSTD_CODEC)
|
|
338
|
+
expect(frame.encoded.dataLength).toBe(frame.chunk.length)
|
|
339
|
+
}
|
|
340
|
+
// Decode each back and verify data integrity.
|
|
341
|
+
for (const frame of frames) {
|
|
342
|
+
const decoded = await decodeStreamPayload(
|
|
343
|
+
{type: "stream.data", encoding: ZSTD_CODEC, dataLength: frame.chunk.length},
|
|
344
|
+
frame.encoded.payload,
|
|
345
|
+
ZSTD_CODEC
|
|
346
|
+
)
|
|
347
|
+
expect(Buffer.from(decoded).equals(Buffer.from(frame.chunk))).toBe(true)
|
|
348
|
+
}
|
|
349
|
+
})
|
|
350
|
+
|
|
351
|
+
it("async zstd encode-decode round-trips independent payloads without leaking state between calls", {skip: !zstdAvailable()}, async () => {
|
|
352
|
+
// Interleave encoding and decoding to prove each zstd frame is independent.
|
|
353
|
+
const payloadA = compressibleChunk(12_000)
|
|
354
|
+
const payloadB = incompressibleChunk(12_000)
|
|
355
|
+
|
|
356
|
+
const encA = await encodeStreamPayload(payloadA, ZSTD_CODEC)
|
|
357
|
+
const encB = await encodeStreamPayload(payloadB, ZSTD_CODEC)
|
|
358
|
+
|
|
359
|
+
// A is compressible, B is not — distinct encodings.
|
|
360
|
+
expect(encA.encoding).toBe(ZSTD_CODEC)
|
|
361
|
+
expect(encB.encoding).toBeUndefined() // raw fallback
|
|
362
|
+
|
|
363
|
+
const decA = await decodeStreamPayload(
|
|
364
|
+
{type: "stream.data", encoding: ZSTD_CODEC, dataLength: payloadA.length},
|
|
365
|
+
encA.payload,
|
|
366
|
+
ZSTD_CODEC
|
|
367
|
+
)
|
|
368
|
+
const decB = await decodeStreamPayload(
|
|
369
|
+
{type: "stream.data"},
|
|
370
|
+
encB.payload,
|
|
390
371
|
ZSTD_CODEC
|
|
391
372
|
)
|
|
392
|
-
assert.deepEqual(decoded, frame.chunk)
|
|
393
|
-
}
|
|
394
|
-
})
|
|
395
373
|
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
const payloadB = incompressibleChunk(12_000)
|
|
400
|
-
|
|
401
|
-
const encA = await encodeStreamPayload(payloadA, ZSTD_CODEC)
|
|
402
|
-
const encB = await encodeStreamPayload(payloadB, ZSTD_CODEC)
|
|
403
|
-
|
|
404
|
-
// A is compressible, B is not — distinct encodings.
|
|
405
|
-
assert.equal(encA.encoding, ZSTD_CODEC)
|
|
406
|
-
assert.equal(encB.encoding, undefined) // raw fallback
|
|
407
|
-
|
|
408
|
-
const decA = await decodeStreamPayload(
|
|
409
|
-
{type: "stream.data", encoding: ZSTD_CODEC, dataLength: payloadA.length},
|
|
410
|
-
encA.payload,
|
|
411
|
-
ZSTD_CODEC
|
|
412
|
-
)
|
|
413
|
-
const decB = await decodeStreamPayload(
|
|
414
|
-
{type: "stream.data"},
|
|
415
|
-
encB.payload,
|
|
416
|
-
ZSTD_CODEC
|
|
417
|
-
)
|
|
418
|
-
|
|
419
|
-
assert.deepEqual(decA, payloadA)
|
|
420
|
-
assert.deepEqual(decB, payloadB)
|
|
374
|
+
expect(Buffer.from(decA).equals(Buffer.from(payloadA))).toBe(true)
|
|
375
|
+
expect(Buffer.from(decB).equals(Buffer.from(payloadB))).toBe(true)
|
|
376
|
+
})
|
|
421
377
|
})
|