socketduct 0.1.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dockerignore +11 -0
- package/.env.example +131 -0
- package/AGENTS.md +168 -0
- package/Dockerfile +30 -0
- package/README.md +134 -0
- package/changelog.d/20260716083144-v2-ack-wire-format.md +3 -0
- package/changelog.d/20260716092044-v2-only-stream-framing.md +1 -0
- package/changelog.d/20260727113000-relay-connect-timeout-destroy.md +1 -0
- package/changelog.d/20260813093000-private-relay-hardening.md +1 -0
- package/changelog.d/20260824183412-initial-target-recovery-https.md +1 -0
- package/changelog.d/20260824230000-tls-relay-session-close.md +1 -0
- package/changelog.d/20260825120000-reconnect-close-target-timeout.md +1 -0
- package/changelog.d/20260829120000-protocol-v3-multiplexing.md +1 -0
- package/changelog.d/20260829160000-v3-review-corrections.md +1 -0
- package/changelog.d/20260830000000-fix-socket-idle-timeout.md +1 -0
- package/changelog.d/20260830090000-multi-gateway-transport-set.md +1 -0
- package/changelog.d/20260830093000-relay-close-settlement.md +1 -0
- package/changelog.d/20260830120000-immutable-gateway-image.md +1 -0
- package/changelog.d/20260902210000-native-reverse-gateway.md +1 -0
- package/changelog.d/20260902230000-relay-resume-terminal-order.md +1 -0
- package/changelog.d/20260902230708-multiplexed-open-ownership.md +1 -0
- package/changelog.d/20260902231541-multiplexed-terminal-close.md +1 -0
- package/changelog.d/20260902231542-reverse-close-position.md +1 -0
- package/changelog.d/20260902231543-reverse-terminal-connect.md +1 -0
- package/changelog.d/20260902233000-reverse-gateway-bounds.md +1 -0
- package/changelog.d/20260903001000-reverse-data-ordering.md +1 -0
- package/changelog.d/20260903003000-reverse-hub-restart-window.md +1 -0
- package/changelog.d/20260903010000-reverse-terminal-lifecycle.md +1 -0
- package/changelog.d/20260904142743-transport-set-readiness-wait.md +1 -0
- package/changelog.d/20260904152855-deferred-open-snapshots-binary-views.md +1 -0
- package/changelog.d/20260904163041-tensorbuzz-container-workflow.md +1 -0
- package/changelog.d/20260904174955-transport-set-terminal-causes.md +1 -0
- package/changelog.d/20260905081000-contain-reverse-auth-write-errors.md +1 -0
- package/changelog.d/20260905100520-reverse-hub-close-race.md +1 -0
- package/changelog.d/20260905120000-persistent-jsonl-logging.md +1 -0
- package/changelog.d/20260908071009-jsonl-partial-append-recovery.md +1 -0
- package/changelog.d/20260908074916-private-log-directory-setup.md +1 -0
- package/changelog.d/20260909093057-reverse-large-write-backpressure.md +1 -0
- package/changelog.d/20260909135830-actionable-error-diagnostics.md +1 -0
- package/changelog.d/20260909150000-transport-set-error-redaction.md +1 -0
- package/docker-compose.dev.yml +19 -0
- package/docker-compose.reverse-gateway.yml +36 -0
- package/docker-compose.yml +73 -0
- package/docs/architecture.md +264 -0
- package/docs/docker-topology.md +164 -0
- package/docs/docker.md +281 -0
- package/docs/logging.md +185 -0
- package/docs/plans/initial-implementation.md +205 -0
- package/docs/plans/socket-docker-integration.md +221 -0
- package/docs/protocol.md +558 -0
- package/docs/releasing.md +18 -0
- package/docs/reverse-gateway.md +186 -0
- package/docs/runbooks/jsonl-integrity-rollout.md +144 -0
- package/docs/running.md +215 -0
- package/docs/security.md +258 -0
- package/docs/session-lifecycle.md +127 -0
- package/docs/socket-api.md +448 -0
- package/eslint.config.js +36 -0
- package/package.json +44 -0
- package/scripts/deploy-gateway.js +194 -0
- package/scripts/test.js +28 -0
- package/src/auth/passwords.js +38 -0
- package/src/auth/static-token-store.js +41 -0
- package/src/auth/user-store.js +139 -0
- package/src/client/connect.js +76 -0
- package/src/client/deferred-virtual-socket.js +312 -0
- package/src/client/http-agent.js +101 -0
- package/src/client/https-agent.js +366 -0
- package/src/client/multiplexed-connection.js +1504 -0
- package/src/client/multiplexed-transport-set.js +917 -0
- package/src/client/multiplexed.js +3 -0
- package/src/client/socketduct-socket.js +1174 -0
- package/src/client/virtual-socket.js +256 -0
- package/src/config/environment.js +400 -0
- package/src/edge/tcp-edge.js +82 -0
- package/src/index.js +101 -0
- package/src/logging/structured-logger.js +745 -0
- package/src/protocol/compression.js +325 -0
- package/src/protocol/frame.js +341 -0
- package/src/protocol/v3-compression.js +44 -0
- package/src/protocol/v3-frame.js +209 -0
- package/src/proxy/multiplexed-relay.js +798 -0
- package/src/proxy/tcp-relay.js +1191 -0
- package/src/relay.js +3 -0
- package/src/reverse/client.js +721 -0
- package/src/reverse/common.js +88 -0
- package/src/reverse/gateway.js +663 -0
- package/src/reverse/hub.js +563 -0
- package/src/reverse/index.js +3 -0
- package/src/reverse/runtime.js +78 -0
- package/src/server/control-handler.js +170 -0
- package/src/server/primary.js +64 -0
- package/src/server/relay-runtime.js +124 -0
- package/src/server/worker.js +39 -0
- package/src/sessions/session.js +212 -0
- package/src/spool/spool-file.js +451 -0
- package/tensorbuzz.yml +36 -0
- package/test/e2e/gateway-process.test.js +262 -0
- package/test/e2e/reverse-gateway-process.test.js +285 -0
- package/test/fixtures/logging/enospc-matrix.mjs +238 -0
- package/test/fixtures/logging/enospc-recovery.mjs +146 -0
- package/test/fixtures/logging/process-writer.mjs +20 -0
- package/test/helpers/tls-certificates.js +115 -0
- package/test/integration/initial-target-recovery.test.js +518 -0
- package/test/integration/logging-recovery.test.js +163 -0
- package/test/integration/multiplexed-compatibility.test.js +254 -0
- package/test/integration/multiplexed-corrections.test.js +551 -0
- package/test/integration/multiplexed-failures.test.js +339 -0
- package/test/integration/multiplexed-transport-set.test.js +456 -0
- package/test/integration/multiplexed-transport.test.js +208 -0
- package/test/integration/reverse-gateway.test.js +1293 -0
- package/test/integration/socketduct-http-agent.test.js +435 -0
- package/test/integration/socketduct-https-agent.test.js +669 -0
- package/test/integration/stream-compression.test.js +1267 -0
- package/test/integration/tcp-forwarding.test.js +2878 -0
- package/test/integration/tls-forwarding.test.js +748 -0
- package/test/unit/auth/auth.test.js +64 -0
- package/test/unit/auth/static-token-store.test.js +14 -0
- package/test/unit/client/connect.test.js +58 -0
- package/test/unit/client/multiplexed-spool-budget.test.js +309 -0
- package/test/unit/client/multiplexed-transport-set.test.js +906 -0
- package/test/unit/client/socketduct-socket.test.js +1124 -0
- package/test/unit/client/virtual-socket.test.js +98 -0
- package/test/unit/config/docker-gateway-files.test.js +274 -0
- package/test/unit/config/environment.test.js +507 -0
- package/test/unit/index.test.js +78 -0
- package/test/unit/logging/structured-logger.test.js +674 -0
- package/test/unit/protocol/compression.test.js +421 -0
- package/test/unit/protocol/frame.test.js +336 -0
- package/test/unit/protocol/v3-compression.test.js +31 -0
- package/test/unit/protocol/v3-frame.test.js +57 -0
- package/test/unit/release-configuration.test.js +21 -0
- package/test/unit/server/control-handler.test.js +291 -0
- package/test/unit/server/primary.test.js +59 -0
- package/test/unit/server/relay-runtime.test.js +434 -0
- package/test/unit/sessions/session.test.js +84 -0
- package/test/unit/spool/spool-file.test.js +295 -0
- package/tsconfig.json +20 -0
|
@@ -0,0 +1,325 @@
|
|
|
1
|
+
import zlib from "node:zlib"
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Frame-local compression for `stream.data` payloads.
|
|
5
|
+
*
|
|
6
|
+
* Every compressed frame is an independent zstd frame: it carries its own
|
|
7
|
+
* uncompressed length (`dataLength`) and shares no dictionary or window with
|
|
8
|
+
* any other frame. That keeps reconnect/resume/replay correct, because a
|
|
9
|
+
* spooled raw chunk can be re-encoded at any time under whatever codec the
|
|
10
|
+
* current connection negotiated (including none).
|
|
11
|
+
*
|
|
12
|
+
* Sequence numbers, acknowledgements, spool offsets, byte limits and
|
|
13
|
+
* backpressure all stay in uncompressed bytes. Compression happens when a
|
|
14
|
+
* frame is built and decompression happens as soon as it is parsed; no other
|
|
15
|
+
* subsystem ever sees a compressed byte.
|
|
16
|
+
* @typedef {import('./frame.js').FrameHeader} FrameHeader
|
|
17
|
+
* @typedef {object} CompressionLimits
|
|
18
|
+
* @property {number} [minCompressBytes] - Payloads smaller than this are sent raw. Default 256.
|
|
19
|
+
* @property {number} [maxCompressedBytes] - Maximum accepted compressed payload size. Default 16 MiB.
|
|
20
|
+
* @property {number} [maxDecompressedBytes] - Maximum accepted decompressed size. Default 16 MiB.
|
|
21
|
+
* @property {number} [maxExpansionRatio] - Maximum accepted dataLength/compressed size ratio. Default 1000.
|
|
22
|
+
* @typedef {object} EncodedPayload
|
|
23
|
+
* @property {Buffer} payload - Bytes to place on the wire.
|
|
24
|
+
* @property {string} [encoding] - Codec name, omitted when the payload is raw.
|
|
25
|
+
* @property {number} [dataLength] - Uncompressed length, present only with an encoding.
|
|
26
|
+
*/
|
|
27
|
+
|
|
28
|
+
export const ZSTD_CODEC = "zstd"
|
|
29
|
+
|
|
30
|
+
const DEFAULT_MIN_COMPRESS_BYTES = 256
|
|
31
|
+
const DEFAULT_MAX_COMPRESSED_BYTES = 16 * 1024 * 1024
|
|
32
|
+
const DEFAULT_MAX_DECOMPRESSED_BYTES = 16 * 1024 * 1024
|
|
33
|
+
const DEFAULT_MAX_EXPANSION_RATIO = 1000
|
|
34
|
+
const COMPRESSION_MARGIN_BYTES = 16
|
|
35
|
+
const ZSTD_LEVEL = 1
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* `node:zlib` only exposes zstd from Node 22.15 onward, but `engines` accepts
|
|
39
|
+
* Node >=22.0, so the binding is probed at runtime rather than imported by name.
|
|
40
|
+
* @typedef {object} ZstdBinding
|
|
41
|
+
* @property {((input: Buffer, options: object, callback: (err: Error | null, result: Buffer) => void) => void) | undefined} zstdCompress - Async zstd compressor callback.
|
|
42
|
+
* @property {((input: Buffer, options: object, callback: (err: Error | null, result: Buffer) => void) => void) | undefined} zstdDecompress - Async zstd decompressor callback.
|
|
43
|
+
* @property {Record<string, number>} constants - zlib constants.
|
|
44
|
+
*/
|
|
45
|
+
const zstd = /** @type {ZstdBinding} */ (/** @type {unknown} */ (zlib))
|
|
46
|
+
|
|
47
|
+
/** Raised when a peer sends metadata or a payload that violates the compression contract. */
|
|
48
|
+
export class CompressionProtocolError extends Error {
|
|
49
|
+
/** @param {string} message - Violation detail. */
|
|
50
|
+
constructor(message) {
|
|
51
|
+
super(message)
|
|
52
|
+
this.name = "CompressionProtocolError"
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Resolves the maximum raw payload size used by the framing and compression
|
|
58
|
+
* paths. This value is also used as a chunk-splitting divisor, so accepting
|
|
59
|
+
* zero, fractions, or unsafe integers would make the send loops unsafe.
|
|
60
|
+
* @param {CompressionLimits} [limits] - Compression size limits.
|
|
61
|
+
* @returns {number} - Maximum permitted decompressed payload size.
|
|
62
|
+
*/
|
|
63
|
+
export function getMaxDecompressedBytes(limits = {}) {
|
|
64
|
+
const maxDecompressedBytes = limits.maxDecompressedBytes ?? DEFAULT_MAX_DECOMPRESSED_BYTES
|
|
65
|
+
if (!Number.isSafeInteger(maxDecompressedBytes) || maxDecompressedBytes < 1) {
|
|
66
|
+
throw new TypeError("compressionLimits.maxDecompressedBytes must be a positive safe integer")
|
|
67
|
+
}
|
|
68
|
+
return maxDecompressedBytes
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** @returns {boolean} - True when this runtime can compress and decompress zstd asynchronously. */
|
|
72
|
+
export function zstdAvailable() {
|
|
73
|
+
return typeof zstd.zstdCompress === "function"
|
|
74
|
+
&& typeof zstd.zstdDecompress === "function"
|
|
75
|
+
&& typeof zstd.constants?.ZSTD_c_compressionLevel === "number"
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** @returns {string[]} - Codecs this runtime can actually use, in preference order. */
|
|
79
|
+
export function availableCodecs() {
|
|
80
|
+
return zstdAvailable() ? [ZSTD_CODEC] : []
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Picks the first mutually supported codec. Any malformed or unknown offer
|
|
85
|
+
* yields no codec, which keeps compact stream records uncompressed.
|
|
86
|
+
* @param {unknown} offer - Codec list advertised by the peer.
|
|
87
|
+
* @param {string[]} supported - Codecs this side supports, in preference order.
|
|
88
|
+
* @returns {string | undefined} - Negotiated codec, or undefined for uncompressed data.
|
|
89
|
+
*/
|
|
90
|
+
export function negotiateCodec(offer, supported) {
|
|
91
|
+
if (!Array.isArray(offer)) {
|
|
92
|
+
return undefined
|
|
93
|
+
}
|
|
94
|
+
for (const codec of supported) {
|
|
95
|
+
if (offer.includes(codec)) {
|
|
96
|
+
return codec
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
return undefined
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* @param {Buffer} input - Raw bytes to compress.
|
|
104
|
+
* @param {object} options - zstd compress options.
|
|
105
|
+
* @returns {Promise<Buffer>} - Compressed buffer.
|
|
106
|
+
*/
|
|
107
|
+
function zstdCompressAsync(input, options) {
|
|
108
|
+
const compress = /** @type {NonNullable<ZstdBinding['zstdCompress']>} */ (zstd.zstdCompress)
|
|
109
|
+
return new Promise((resolve, reject) => {
|
|
110
|
+
compress(input, options, (err, result) => {
|
|
111
|
+
if (err) {
|
|
112
|
+
reject(err)
|
|
113
|
+
return
|
|
114
|
+
}
|
|
115
|
+
resolve(result)
|
|
116
|
+
})
|
|
117
|
+
})
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* @param {Buffer} input - Compressed bytes.
|
|
122
|
+
* @param {object} options - zstd decompress options.
|
|
123
|
+
* @returns {Promise<Buffer>} - Decompressed buffer.
|
|
124
|
+
*/
|
|
125
|
+
function zstdDecompressAsync(input, options) {
|
|
126
|
+
const decompress = /** @type {NonNullable<ZstdBinding['zstdDecompress']>} */ (zstd.zstdDecompress)
|
|
127
|
+
return new Promise((resolve, reject) => {
|
|
128
|
+
decompress(input, options, (err, result) => {
|
|
129
|
+
if (err) {
|
|
130
|
+
reject(err)
|
|
131
|
+
return
|
|
132
|
+
}
|
|
133
|
+
resolve(result)
|
|
134
|
+
})
|
|
135
|
+
})
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* Encodes one `stream.data` payload, falling back to raw whenever compression
|
|
140
|
+
* would not pay for itself. The returned frame is never larger than the raw one.
|
|
141
|
+
* @param {Buffer} chunk - Raw payload bytes.
|
|
142
|
+
* @param {string | undefined} codec - Negotiated codec, or undefined for raw.
|
|
143
|
+
* @param {AdaptiveCompressor} [adaptive] - Per-direction adaptive state.
|
|
144
|
+
* @param {CompressionLimits} [limits] - Compression limits.
|
|
145
|
+
* @returns {Promise<EncodedPayload>} - Wire payload plus optional metadata.
|
|
146
|
+
*/
|
|
147
|
+
export async function encodeStreamPayload(chunk, codec, adaptive, limits = {}) {
|
|
148
|
+
const minCompressBytes = limits.minCompressBytes ?? DEFAULT_MIN_COMPRESS_BYTES
|
|
149
|
+
const maxCompressedBytes = limits.maxCompressedBytes ?? DEFAULT_MAX_COMPRESSED_BYTES
|
|
150
|
+
const maxDecompressedBytes = getMaxDecompressedBytes(limits)
|
|
151
|
+
|
|
152
|
+
// A chunk over the uncompressed limit is rejected by the peer either way, so
|
|
153
|
+
// there is no point spending CPU compressing it.
|
|
154
|
+
if (codec !== ZSTD_CODEC || !zstdAvailable() || chunk.length < minCompressBytes || chunk.length > maxDecompressedBytes) {
|
|
155
|
+
return {payload: chunk}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
if (adaptive !== undefined && !adaptive.shouldCompress()) {
|
|
159
|
+
return {payload: chunk}
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
const levelParam = /** @type {number} */ (zstd.constants.ZSTD_c_compressionLevel)
|
|
163
|
+
/** @type {Buffer} */
|
|
164
|
+
let compressed
|
|
165
|
+
try {
|
|
166
|
+
compressed = await zstdCompressAsync(chunk, {params: {[levelParam]: ZSTD_LEVEL}})
|
|
167
|
+
} catch {
|
|
168
|
+
adaptive?.record(1)
|
|
169
|
+
return {payload: chunk}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
adaptive?.record(compressed.length / chunk.length)
|
|
173
|
+
|
|
174
|
+
// Expansion defense: only keep the compressed form when it actually wins.
|
|
175
|
+
if (compressed.length + COMPRESSION_MARGIN_BYTES >= chunk.length || compressed.length > maxCompressedBytes) {
|
|
176
|
+
return {payload: chunk}
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
return {payload: compressed, encoding: ZSTD_CODEC, dataLength: chunk.length}
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* Validates compression metadata and returns the raw payload bytes.
|
|
184
|
+
*
|
|
185
|
+
* Fails closed: any malformed metadata, unnegotiated codec, out-of-bounds
|
|
186
|
+
* length, excessive expansion ratio or corrupt payload throws. There is no
|
|
187
|
+
* fallback to treating a compressed payload as raw.
|
|
188
|
+
* @param {FrameHeader} header - Incoming `stream.data` header.
|
|
189
|
+
* @param {Buffer} payload - Incoming wire payload.
|
|
190
|
+
* @param {string | undefined} codec - Codec negotiated on this connection.
|
|
191
|
+
* @param {CompressionLimits} [limits] - Decompression bounds.
|
|
192
|
+
* @returns {Promise<Buffer>} - Uncompressed payload bytes.
|
|
193
|
+
*/
|
|
194
|
+
export async function decodeStreamPayload(header, payload, codec, limits = {}) {
|
|
195
|
+
const encoding = header.encoding
|
|
196
|
+
const dataLength = header.dataLength
|
|
197
|
+
const maxUncompressedBytes = getMaxDecompressedBytes(limits)
|
|
198
|
+
|
|
199
|
+
if (encoding === undefined || encoding === null) {
|
|
200
|
+
if (dataLength !== undefined) {
|
|
201
|
+
throw new CompressionProtocolError("stream.data dataLength requires an encoding.")
|
|
202
|
+
}
|
|
203
|
+
// The uncompressed-size limit is a property of the stream, not of the codec:
|
|
204
|
+
// an uncompressed record that exceeds it is just as much a violation as a compressed one.
|
|
205
|
+
if (payload.length > maxUncompressedBytes) {
|
|
206
|
+
throw new CompressionProtocolError(`stream.data payload ${payload.length} exceeds the maximum of ${maxUncompressedBytes} bytes.`)
|
|
207
|
+
}
|
|
208
|
+
return payload
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
if (typeof encoding !== "string") {
|
|
212
|
+
throw new CompressionProtocolError("stream.data encoding must be a string.")
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
if (codec === undefined) {
|
|
216
|
+
throw new CompressionProtocolError(`stream.data encoding "${encoding}" was not negotiated for this connection.`)
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
if (encoding !== codec) {
|
|
220
|
+
throw new CompressionProtocolError(`stream.data encoding "${encoding}" does not match negotiated codec "${codec}".`)
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
if (!zstdAvailable()) {
|
|
224
|
+
throw new CompressionProtocolError(`stream.data encoding "${encoding}" is not supported by this runtime.`)
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
const maxCompressedBytes = limits.maxCompressedBytes ?? DEFAULT_MAX_COMPRESSED_BYTES
|
|
228
|
+
const maxExpansionRatio = limits.maxExpansionRatio ?? DEFAULT_MAX_EXPANSION_RATIO
|
|
229
|
+
|
|
230
|
+
if (typeof dataLength !== "number" || !Number.isSafeInteger(dataLength) || dataLength <= 0) {
|
|
231
|
+
throw new CompressionProtocolError("stream.data dataLength must be a positive safe integer.")
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
if (dataLength > maxUncompressedBytes) {
|
|
235
|
+
throw new CompressionProtocolError(`stream.data dataLength ${dataLength} exceeds the maximum of ${maxUncompressedBytes} bytes.`)
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
if (payload.length === 0) {
|
|
239
|
+
throw new CompressionProtocolError("stream.data compressed payload must not be empty.")
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
if (payload.length > maxCompressedBytes) {
|
|
243
|
+
throw new CompressionProtocolError(`stream.data compressed payload ${payload.length} exceeds the maximum of ${maxCompressedBytes} bytes.`)
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
if (dataLength / payload.length > maxExpansionRatio) {
|
|
247
|
+
throw new CompressionProtocolError(`stream.data expansion ratio exceeds the maximum of ${maxExpansionRatio}.`)
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
/** @type {Buffer} */
|
|
251
|
+
let decompressed
|
|
252
|
+
try {
|
|
253
|
+
// The advertised length is also the hard output bound, so a bomb can never
|
|
254
|
+
// produce more bytes than the sender promised.
|
|
255
|
+
decompressed = await zstdDecompressAsync(payload, {maxOutputLength: dataLength})
|
|
256
|
+
} catch (error) {
|
|
257
|
+
const detail = error instanceof Error ? error.message : String(error)
|
|
258
|
+
throw new CompressionProtocolError(`stream.data payload failed to decompress: ${detail}`)
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
if (decompressed.length !== dataLength) {
|
|
262
|
+
throw new CompressionProtocolError(`stream.data decompressed length ${decompressed.length} does not match the advertised dataLength ${dataLength}.`)
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
return decompressed
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/**
|
|
269
|
+
* Send-side only: stops paying CPU on streams that do not compress (TLS, media,
|
|
270
|
+
* already-compressed payloads) and periodically re-probes in case the stream
|
|
271
|
+
* changes character. Receivers need no knowledge of this, because every frame
|
|
272
|
+
* declares its own encoding.
|
|
273
|
+
*/
|
|
274
|
+
export class AdaptiveCompressor {
|
|
275
|
+
/** @type {number} */
|
|
276
|
+
#ratio = 0
|
|
277
|
+
|
|
278
|
+
/** @type {number} */
|
|
279
|
+
#skipRemaining = 0
|
|
280
|
+
|
|
281
|
+
/** @type {number} */
|
|
282
|
+
#backoff = 0
|
|
283
|
+
|
|
284
|
+
/** @type {number} */
|
|
285
|
+
#incompressibleRatio
|
|
286
|
+
|
|
287
|
+
/** @type {number} */
|
|
288
|
+
#maxSkipChunks
|
|
289
|
+
|
|
290
|
+
/**
|
|
291
|
+
* @param {object} [options] - Adaptive options.
|
|
292
|
+
* @param {number} [options.incompressibleRatio] - Ratio at or above which a stream counts as incompressible. Default 0.95.
|
|
293
|
+
* @param {number} [options.maxSkipChunks] - Upper bound on consecutive skipped chunks. Default 64.
|
|
294
|
+
*/
|
|
295
|
+
constructor(options = {}) {
|
|
296
|
+
this.#incompressibleRatio = options.incompressibleRatio ?? 0.95
|
|
297
|
+
this.#maxSkipChunks = options.maxSkipChunks ?? 64
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
/** @returns {boolean} - True when the next eligible chunk should be compressed. */
|
|
301
|
+
shouldCompress() {
|
|
302
|
+
if (this.#skipRemaining > 0) {
|
|
303
|
+
this.#skipRemaining -= 1
|
|
304
|
+
return false
|
|
305
|
+
}
|
|
306
|
+
return true
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
/**
|
|
310
|
+
* @param {number} ratio - Achieved compressed/uncompressed ratio.
|
|
311
|
+
* @returns {void}
|
|
312
|
+
*/
|
|
313
|
+
record(ratio) {
|
|
314
|
+
this.#ratio = this.#ratio === 0 ? ratio : (this.#ratio * 0.7) + (ratio * 0.3)
|
|
315
|
+
|
|
316
|
+
if (this.#ratio >= this.#incompressibleRatio) {
|
|
317
|
+
this.#backoff = this.#backoff === 0 ? 1 : Math.min(this.#backoff * 2, this.#maxSkipChunks)
|
|
318
|
+
this.#skipRemaining = this.#backoff
|
|
319
|
+
return
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
this.#backoff = 0
|
|
323
|
+
this.#skipRemaining = 0
|
|
324
|
+
}
|
|
325
|
+
}
|
|
@@ -0,0 +1,341 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @typedef {Record<string, unknown> & {type: string}} FrameHeader
|
|
3
|
+
* @typedef {{header: FrameHeader, payload: Buffer, wireFormat: "compact" | "envelope"}} Frame
|
|
4
|
+
* @typedef {{maxHeaderBytes?: number, maxPayloadBytes?: number, maxInputBytes?: number, compactDirection?: "client-to-target" | "target-to-client"}} FrameDecoderOptions
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
const DEFAULT_MAX_HEADER_BYTES = 64 * 1024
|
|
8
|
+
const DEFAULT_MAX_PAYLOAD_BYTES = 16 * 1024 * 1024
|
|
9
|
+
const UINT32_BYTES = 4
|
|
10
|
+
const COMPACT_DATA_RAW = 0x80
|
|
11
|
+
const COMPACT_DATA_ZSTD = 0x81
|
|
12
|
+
const COMPACT_ACK = 0x82
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* @param {number} value - Safe non-negative integer.
|
|
16
|
+
* @returns {Buffer} Canonical unsigned LEB128.
|
|
17
|
+
*/
|
|
18
|
+
function encodeVarint(value) {
|
|
19
|
+
if (!Number.isSafeInteger(value) || value < 0) throw new RangeError("Compact varint value must be a non-negative safe integer")
|
|
20
|
+
const bytes = []
|
|
21
|
+
do {
|
|
22
|
+
const low = value % 128
|
|
23
|
+
value = Math.floor(value / 128)
|
|
24
|
+
bytes.push(low | (value > 0 ? 0x80 : 0))
|
|
25
|
+
} while (value > 0)
|
|
26
|
+
return Buffer.from(bytes)
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* @param {{sequence: number, payload: Uint8Array, dataLength?: number}} record - Compact stream record.
|
|
31
|
+
* @returns {Buffer} Encoded record.
|
|
32
|
+
*/
|
|
33
|
+
export function encodeCompactData(record) {
|
|
34
|
+
const payload = Buffer.from(record.payload)
|
|
35
|
+
const parts = [Buffer.from([record.dataLength === undefined ? COMPACT_DATA_RAW : COMPACT_DATA_ZSTD]), encodeVarint(record.sequence)]
|
|
36
|
+
if (record.dataLength !== undefined) parts.push(encodeVarint(record.dataLength))
|
|
37
|
+
parts.push(encodeVarint(payload.length), payload)
|
|
38
|
+
return Buffer.concat(parts)
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* @param {number} sequence - Acknowledged byte sequence.
|
|
43
|
+
* @returns {Buffer} Encoded record.
|
|
44
|
+
*/
|
|
45
|
+
export function encodeCompactAck(sequence) {
|
|
46
|
+
return Buffer.concat([Buffer.from([COMPACT_ACK]), encodeVarint(sequence)])
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* @param {FrameHeader} header - header value.
|
|
51
|
+
* @param {Uint8Array} [payload] - payload value.
|
|
52
|
+
* @returns {Buffer} - Result value.
|
|
53
|
+
*/
|
|
54
|
+
export function encodeFrame(header, payload = Buffer.alloc(0)) {
|
|
55
|
+
const headerBytes = Buffer.from(JSON.stringify(header), "utf8")
|
|
56
|
+
const payloadBytes = Buffer.from(payload)
|
|
57
|
+
const frame = Buffer.allocUnsafe(UINT32_BYTES + headerBytes.length + UINT32_BYTES + payloadBytes.length)
|
|
58
|
+
|
|
59
|
+
frame.writeUInt32BE(headerBytes.length, 0)
|
|
60
|
+
headerBytes.copy(frame, UINT32_BYTES)
|
|
61
|
+
frame.writeUInt32BE(payloadBytes.length, UINT32_BYTES + headerBytes.length)
|
|
62
|
+
payloadBytes.copy(frame, UINT32_BYTES + headerBytes.length + UINT32_BYTES)
|
|
63
|
+
|
|
64
|
+
return frame
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export class FrameDecoder {
|
|
68
|
+
/** @type {number} */
|
|
69
|
+
#maxHeaderBytes
|
|
70
|
+
|
|
71
|
+
/** @type {number} */
|
|
72
|
+
#maxPayloadBytes
|
|
73
|
+
|
|
74
|
+
/** @type {number} */
|
|
75
|
+
#maxInputBytes
|
|
76
|
+
|
|
77
|
+
/** @type {number} */
|
|
78
|
+
#inputBytes = 0
|
|
79
|
+
|
|
80
|
+
/** @type {"client-to-target" | "target-to-client" | undefined} */
|
|
81
|
+
#compactDirection
|
|
82
|
+
|
|
83
|
+
/** @type {Buffer} */
|
|
84
|
+
#buffer = Buffer.alloc(0)
|
|
85
|
+
|
|
86
|
+
/** @type {number} */
|
|
87
|
+
#readPos = 0
|
|
88
|
+
|
|
89
|
+
/** @type {number} */
|
|
90
|
+
#writePos = 0
|
|
91
|
+
|
|
92
|
+
/** @param {FrameDecoderOptions} [options] - Decoder limits. */
|
|
93
|
+
constructor(options = {}) {
|
|
94
|
+
this.#maxHeaderBytes = options.maxHeaderBytes ?? DEFAULT_MAX_HEADER_BYTES
|
|
95
|
+
this.#maxPayloadBytes = options.maxPayloadBytes ?? DEFAULT_MAX_PAYLOAD_BYTES
|
|
96
|
+
this.#maxInputBytes = options.maxInputBytes ?? 0
|
|
97
|
+
this.#compactDirection = options.compactDirection
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* @param {Uint8Array} chunk - chunk value.
|
|
102
|
+
* @returns {Frame[]} - Result value.
|
|
103
|
+
*/
|
|
104
|
+
push(chunk) {
|
|
105
|
+
const chunkLen = chunk.length
|
|
106
|
+
if (chunkLen === 0) {
|
|
107
|
+
return []
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
this.#inputBytes += chunkLen
|
|
111
|
+
if (this.#maxInputBytes > 0 && this.#inputBytes > this.#maxInputBytes) {
|
|
112
|
+
throw new Error(`Input byte budget exceeded: ${this.#inputBytes} > ${this.#maxInputBytes}`)
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
this.#maybeGrow(chunkLen)
|
|
116
|
+
this.#copyChunk(chunk, chunkLen)
|
|
117
|
+
|
|
118
|
+
return this.#extractFrames()
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* @returns {void}
|
|
123
|
+
*/
|
|
124
|
+
disableInputBudget() {
|
|
125
|
+
this.#maxInputBytes = 0
|
|
126
|
+
this.#inputBytes = 0
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* @param {number} needed - Additional bytes needed.
|
|
131
|
+
*/
|
|
132
|
+
#maybeGrow(needed) {
|
|
133
|
+
let writePos = this.#writePos
|
|
134
|
+
const buffer = this.#buffer
|
|
135
|
+
let minCapacity = writePos + needed
|
|
136
|
+
|
|
137
|
+
if (minCapacity <= buffer.length) {
|
|
138
|
+
return
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
const readPos = this.#readPos
|
|
142
|
+
const used = writePos - readPos
|
|
143
|
+
|
|
144
|
+
if (readPos > 0) {
|
|
145
|
+
buffer.copyWithin(0, readPos, writePos)
|
|
146
|
+
this.#readPos = 0
|
|
147
|
+
this.#writePos = used
|
|
148
|
+
writePos = used
|
|
149
|
+
minCapacity = writePos + needed
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
if (minCapacity <= this.#buffer.length) {
|
|
153
|
+
return
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
let newCapacity = this.#buffer.length * 2
|
|
157
|
+
if (newCapacity < 256) {
|
|
158
|
+
newCapacity = 256
|
|
159
|
+
}
|
|
160
|
+
while (newCapacity < minCapacity) {
|
|
161
|
+
newCapacity = Math.floor(newCapacity * 1.5)
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
const newBuf = Buffer.allocUnsafe(newCapacity)
|
|
165
|
+
if (used > 0) {
|
|
166
|
+
newBuf.set(this.#buffer.subarray(0, used), 0)
|
|
167
|
+
}
|
|
168
|
+
this.#buffer = newBuf
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* @param {Uint8Array} chunk - chunk value.
|
|
173
|
+
* @param {number} chunkLen - chunk length.
|
|
174
|
+
*/
|
|
175
|
+
#copyChunk(chunk, chunkLen) {
|
|
176
|
+
const writePos = this.#writePos
|
|
177
|
+
const buf = this.#buffer
|
|
178
|
+
|
|
179
|
+
if (chunk instanceof Buffer) {
|
|
180
|
+
buf.set(chunk, writePos)
|
|
181
|
+
} else {
|
|
182
|
+
buf.set(Buffer.from(chunk), writePos)
|
|
183
|
+
}
|
|
184
|
+
this.#writePos = writePos + chunkLen
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/**
|
|
188
|
+
* @returns {Frame[]} - Result value.
|
|
189
|
+
*/
|
|
190
|
+
#extractFrames() {
|
|
191
|
+
/** @type {Frame[]} */
|
|
192
|
+
const frames = []
|
|
193
|
+
const buffer = this.#buffer
|
|
194
|
+
let readPos = this.#readPos
|
|
195
|
+
const writePos = this.#writePos
|
|
196
|
+
|
|
197
|
+
while (true) {
|
|
198
|
+
const remaining = writePos - readPos
|
|
199
|
+
if (remaining === 0) {
|
|
200
|
+
break
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
const firstByte = buffer[readPos]
|
|
204
|
+
if (this.#compactDirection !== undefined && (firstByte === COMPACT_DATA_RAW || firstByte === COMPACT_DATA_ZSTD || firstByte === COMPACT_ACK)) {
|
|
205
|
+
const compact = this.#parseCompact(readPos, writePos)
|
|
206
|
+
if (compact === undefined) break
|
|
207
|
+
frames.push(compact.frame)
|
|
208
|
+
readPos = compact.end
|
|
209
|
+
continue
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
if (remaining < UINT32_BYTES) break
|
|
213
|
+
|
|
214
|
+
const headerLength = buffer.readUInt32BE(readPos)
|
|
215
|
+
if (headerLength <= 0 || headerLength > this.#maxHeaderBytes) {
|
|
216
|
+
throw new Error(`Invalid frame header length: ${headerLength}`)
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
const payloadLengthOffset = readPos + UINT32_BYTES + headerLength
|
|
220
|
+
if (remaining < payloadLengthOffset + UINT32_BYTES - readPos) {
|
|
221
|
+
break
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
const payloadLength = buffer.readUInt32BE(payloadLengthOffset)
|
|
225
|
+
if (payloadLength > this.#maxPayloadBytes) {
|
|
226
|
+
throw new Error(`Invalid frame payload length: ${payloadLength}`)
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
const frameEnd = payloadLengthOffset + UINT32_BYTES + payloadLength
|
|
230
|
+
if (remaining < frameEnd - readPos) {
|
|
231
|
+
break
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
const headerStart = readPos + UINT32_BYTES
|
|
235
|
+
const headerBytes = buffer.subarray(headerStart, payloadLengthOffset)
|
|
236
|
+
const payload = buffer.subarray(payloadLengthOffset + UINT32_BYTES, frameEnd)
|
|
237
|
+
const header = parseHeader(headerBytes)
|
|
238
|
+
|
|
239
|
+
frames.push({header, payload: Buffer.from(payload), wireFormat: "envelope"})
|
|
240
|
+
|
|
241
|
+
readPos = frameEnd
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
this.#readPos = readPos
|
|
245
|
+
|
|
246
|
+
if (readPos > 0 && readPos >= writePos) {
|
|
247
|
+
this.#readPos = 0
|
|
248
|
+
this.#writePos = 0
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
return frames
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
/**
|
|
255
|
+
* @param {number} start - Record start.
|
|
256
|
+
* @param {number} end - Available end.
|
|
257
|
+
* @returns {{frame: Frame, end: number} | undefined} Parsed record or undefined when incomplete.
|
|
258
|
+
*/
|
|
259
|
+
#parseCompact(start, end) {
|
|
260
|
+
const tag = this.#buffer[start]
|
|
261
|
+
if (tag !== COMPACT_DATA_RAW && tag !== COMPACT_DATA_ZSTD && tag !== COMPACT_ACK) throw new Error(`Invalid compact frame tag: ${tag}`)
|
|
262
|
+
let offset = start + 1
|
|
263
|
+
const sequenceResult = this.#readVarint(offset, end)
|
|
264
|
+
if (sequenceResult === undefined) return undefined
|
|
265
|
+
const sequence = sequenceResult.value
|
|
266
|
+
offset = sequenceResult.end
|
|
267
|
+
const dataDirection = /** @type {"client-to-target" | "target-to-client"} */ (this.#compactDirection)
|
|
268
|
+
const ackDirection = dataDirection === "client-to-target" ? "target-to-client" : "client-to-target"
|
|
269
|
+
if (tag === COMPACT_ACK) {
|
|
270
|
+
return {frame: {header: {type: "stream.ack", direction: ackDirection, ackSequence: sequence, compact: true}, payload: Buffer.alloc(0), wireFormat: "compact"}, end: offset}
|
|
271
|
+
}
|
|
272
|
+
let dataLength
|
|
273
|
+
if (tag === COMPACT_DATA_ZSTD) {
|
|
274
|
+
const lengthResult = this.#readVarint(offset, end)
|
|
275
|
+
if (lengthResult === undefined) return undefined
|
|
276
|
+
dataLength = lengthResult.value
|
|
277
|
+
offset = lengthResult.end
|
|
278
|
+
}
|
|
279
|
+
const payloadResult = this.#readVarint(offset, end)
|
|
280
|
+
if (payloadResult === undefined) return undefined
|
|
281
|
+
const payloadLength = payloadResult.value
|
|
282
|
+
offset = payloadResult.end
|
|
283
|
+
if (payloadLength > this.#maxPayloadBytes) throw new Error(`Invalid frame payload length: ${payloadLength}`)
|
|
284
|
+
if (end - offset < payloadLength) return undefined
|
|
285
|
+
/** @type {FrameHeader} */
|
|
286
|
+
const header = {type: "stream.data", direction: dataDirection, sequence, compact: true}
|
|
287
|
+
if (tag === COMPACT_DATA_ZSTD) {
|
|
288
|
+
header.encoding = "zstd"
|
|
289
|
+
header.dataLength = dataLength
|
|
290
|
+
}
|
|
291
|
+
return {frame: {header, payload: Buffer.from(this.#buffer.subarray(offset, offset + payloadLength)), wireFormat: "compact"}, end: offset + payloadLength}
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
/**
|
|
295
|
+
* @param {number} start - Varint start.
|
|
296
|
+
* @param {number} end - Available end.
|
|
297
|
+
* @returns {{value: number, end: number} | undefined} Parsed value or undefined when incomplete.
|
|
298
|
+
*/
|
|
299
|
+
#readVarint(start, end) {
|
|
300
|
+
let value = 0
|
|
301
|
+
let multiplier = 1
|
|
302
|
+
for (let offset = start; offset < end && offset < start + 8; offset += 1) {
|
|
303
|
+
const byte = /** @type {number} */ (this.#buffer[offset])
|
|
304
|
+
const digit = byte & 0x7f
|
|
305
|
+
if (digit > Math.floor((Number.MAX_SAFE_INTEGER - value) / multiplier)) throw new Error("Compact varint exceeds safe integer range")
|
|
306
|
+
value += digit * multiplier
|
|
307
|
+
if ((byte & 0x80) === 0) {
|
|
308
|
+
if (offset > start && digit === 0) throw new Error("Compact varint is not canonical")
|
|
309
|
+
return {value, end: offset + 1}
|
|
310
|
+
}
|
|
311
|
+
multiplier *= 128
|
|
312
|
+
}
|
|
313
|
+
if (end - start >= 8) throw new Error("Compact varint exceeds safe integer range")
|
|
314
|
+
return undefined
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
/**
|
|
319
|
+
* @param {Buffer} headerBytes - headerBytes value.
|
|
320
|
+
* @returns {FrameHeader} - Result value.
|
|
321
|
+
*/
|
|
322
|
+
function parseHeader(headerBytes) {
|
|
323
|
+
const parsed = JSON.parse(headerBytes.toString("utf8"))
|
|
324
|
+
|
|
325
|
+
if (!isFrameHeader(parsed)) {
|
|
326
|
+
throw new Error("Invalid frame header: expected object with string type")
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
return parsed
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
/**
|
|
333
|
+
* @param {unknown} value - value value.
|
|
334
|
+
* @returns {value is FrameHeader} - Result value.
|
|
335
|
+
*/
|
|
336
|
+
function isFrameHeader(value) {
|
|
337
|
+
return typeof value === "object"
|
|
338
|
+
&& value !== null
|
|
339
|
+
&& !Array.isArray(value)
|
|
340
|
+
&& typeof /** @type {{type?: unknown}} */ (value).type === "string"
|
|
341
|
+
}
|