tunnelfetch 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +143 -9
- package/README.zh-CN.md +84 -4
- package/package.json +2 -2
- package/src/client/decode.js +103 -5
- package/src/client.js +96 -9
- package/src/http2/connection.js +1 -1
- package/src/pool.js +20 -4
- package/src/tls/record.js +13 -5
- package/types/client/decode.d.ts +32 -2
- package/types/client.d.ts +37 -94
- package/types/http2/connection.d.ts +2 -2
- package/types/tls/record.d.ts +31 -23
package/README.md
CHANGED
|
@@ -2,11 +2,23 @@
|
|
|
2
2
|
|
|
3
3
|
[English](README.md) · [简体中文](README.zh-CN.md)
|
|
4
4
|
|
|
5
|
+
[](https://github.com/latentharbor/tunnelfetch/actions/workflows/ci.yml)
|
|
6
|
+
[](https://www.npmjs.com/package/tunnelfetch)
|
|
7
|
+
[](LICENSE)
|
|
8
|
+
|
|
9
|
+
|
|
5
10
|
A `fetch`-shaped HTTP client that can route through an HTTP CONNECT, HTTPS, or SOCKS5 proxy on
|
|
6
11
|
runtimes that expose only raw TCP — principally Cloudflare Workers (`workerd`).
|
|
7
12
|
|
|
8
13
|
Zero dependencies. ESM. No build step. No `node:` imports anywhere in `src/`.
|
|
9
14
|
|
|
15
|
+
> **Maturity.** This is a new implementation, not a battle-tested one. It implements TLS 1.2/1.3 and
|
|
16
|
+
> certificate validation in userland — a category where good tests are necessary and not sufficient.
|
|
17
|
+
> It has 1132 hermetic tests, RFC vectors, byte-by-byte fragmentation, live edge interop, seeded
|
|
18
|
+
> fuzzing of every peer-facing parser, and 95% line coverage. It has **not** had an external
|
|
19
|
+
> security audit. Treat it as a high-quality implementation worth trying, not as something proven
|
|
20
|
+
> in production. Please report anything you find — see [SECURITY.md](SECURITY.md).
|
|
21
|
+
|
|
10
22
|
```js
|
|
11
23
|
import { Client } from 'tunnelfetch';
|
|
12
24
|
import { connect } from 'cloudflare:sockets';
|
|
@@ -203,6 +215,69 @@ duration — raise it above your feed's heartbeat interval, or a quiet-but-alive
|
|
|
203
215
|
Abandoning a stream part-way never returns the connection to the pool: its position is unknown,
|
|
204
216
|
and reusing it would splice the remains of one response onto the next request.
|
|
205
217
|
|
|
218
|
+
### `br`, `zstd`, and other codings
|
|
219
|
+
|
|
220
|
+
`gzip` and `deflate` are built in because the runtime decompresses them natively. Anything else is
|
|
221
|
+
pluggable: give `decoders` a function per coding and it is appended to `Accept-Encoding` and applied
|
|
222
|
+
to matching responses. Registering is what makes advertising honest — asking for a coding you cannot
|
|
223
|
+
read turns every such response into garbage, so the two move together and cannot drift.
|
|
224
|
+
|
|
225
|
+
```js
|
|
226
|
+
import { Client } from 'tunnelfetch';
|
|
227
|
+
import { connect } from 'cloudflare:sockets';
|
|
228
|
+
import { BrotliDecStream, BrotliStreamResultCode, initSync } from 'brotli-dec-wasm/web';
|
|
229
|
+
import wasm from 'brotli-dec-wasm/web/bg.wasm';
|
|
230
|
+
|
|
231
|
+
// Module scope, so instantiation lands in isolate startup, which this runtime does not bill.
|
|
232
|
+
// Measured on the edge: it costs nothing detectable (12 ms startup with it, 12 ms without).
|
|
233
|
+
initSync({ module: wasm });
|
|
234
|
+
|
|
235
|
+
const brotli = (stream) => {
|
|
236
|
+
const dec = new BrotliDecStream();
|
|
237
|
+
return stream.pipeThrough(new TransformStream({
|
|
238
|
+
transform(chunk, c) {
|
|
239
|
+
let r = dec.dec(chunk, 1 << 20);
|
|
240
|
+
if (r.buf.length) c.enqueue(r.buf);
|
|
241
|
+
while (r.code === BrotliStreamResultCode.NeedsMoreOutput) {
|
|
242
|
+
r = dec.dec(new Uint8Array(0), 1 << 20);
|
|
243
|
+
if (r.buf.length) c.enqueue(r.buf);
|
|
244
|
+
}
|
|
245
|
+
},
|
|
246
|
+
}));
|
|
247
|
+
};
|
|
248
|
+
|
|
249
|
+
const client = new Client({ connect, proxy, decoders: { br: brotli } });
|
|
250
|
+
// Now sends `Accept-Encoding: gzip, deflate, br` and decodes `Content-Encoding: br`.
|
|
251
|
+
```
|
|
252
|
+
|
|
253
|
+
Order is registration order after the built-ins, so `{ br, zstd }` produces exactly the
|
|
254
|
+
`gzip, deflate, br, zstd` a Chrome sends — which is the actual reason to do this. This client
|
|
255
|
+
presents curl's TLS and HTTP/2 fingerprints by default, and `gzip, deflate` is what curl sends, so
|
|
256
|
+
the default is already consistent. It stops being consistent the moment you dress the handshake up
|
|
257
|
+
as a browser and leave the header behind.
|
|
258
|
+
|
|
259
|
+
It is not a saving. Measured on the edge, decoding the same 256 KB page to the same bytes:
|
|
260
|
+
|
|
261
|
+
| Implementation | Algorithm | ms/MB |
|
|
262
|
+
|---|---|---|
|
|
263
|
+
| `DecompressionStream` — the runtime's own C++ | inflate | **2.5** |
|
|
264
|
+
| WASM brotli (`brotli-dec-wasm`) | brotli | 4.7 |
|
|
265
|
+
| JS inflate (`fflate` / `pako`) | inflate | 7.5 / 8.2 |
|
|
266
|
+
| JS brotli (`brotli`) | brotli | 19.7 |
|
|
267
|
+
|
|
268
|
+
Two things fall out, and both are worth knowing before reaching for WebAssembly anywhere else in a
|
|
269
|
+
Worker. **WASM is about 4x faster than JavaScript at the same algorithm** (brotli: 4.7 against
|
|
270
|
+
19.7) — so if a coding has no native path, WASM is the right way to add one. And **native is about
|
|
271
|
+
3x faster than JavaScript at the same algorithm** (inflate: 2.5 against 7.5–8.2) — so where a
|
|
272
|
+
native path already exists, nothing in userland improves on it. That is why `gzip` and `deflate`
|
|
273
|
+
are not overridable: replacing them could only ever be slower, and doing it silently is the kind of
|
|
274
|
+
quiet downgrade this package refuses everywhere else.
|
|
275
|
+
|
|
276
|
+
Brotli itself lands at 1.9x native inflate. That gap is the price of the coding, and the wire bytes
|
|
277
|
+
it saves do not pay it back — see [What this cannot do](#what-this-cannot-do-and-why). Decoder names
|
|
278
|
+
are validated as HTTP tokens, a decoder that throws fails the body closed rather than truncating it,
|
|
279
|
+
and an unregistered coding is still refused.
|
|
280
|
+
|
|
206
281
|
### HTTP/2 — access, not speed
|
|
207
282
|
|
|
208
283
|
The client offers `h2` and `http/1.1` in ALPN by default and speaks whichever the server selects.
|
|
@@ -377,8 +452,10 @@ Not implemented, and not planned:
|
|
|
377
452
|
- **ChaCha20-Poly1305.** It buys nothing: a server can only pick a suite we offered, TLS 1.3
|
|
378
453
|
mandates AES-128-GCM, and AES-GCM is universal in TLS 1.2 deployments. WebCrypto has no
|
|
379
454
|
ChaCha20, so offering it would mean a `node:crypto` dependency for zero compatibility gain.
|
|
380
|
-
- **Client certificates (mTLS),
|
|
381
|
-
|
|
455
|
+
- **Client certificates (mTLS), 0-RTT, renegotiation.** A `HelloRequest` is refused rather than
|
|
456
|
+
honoured. 0-RTT is a decision rather than an omission: early data can be replayed, so offering
|
|
457
|
+
it would let an attacker who captured a POST replay it. (Session resumption itself *is*
|
|
458
|
+
implemented — a ticket is kept per pool key and offered with `psk_dhe_ke`.)
|
|
382
459
|
- **Revocation fetching (CRL downloads, OCSP responder queries).** Both need network round trips
|
|
383
460
|
mid-handshake, through the proxy, and an OCSP query tells the CA which origins you visit.
|
|
384
461
|
Revocation *is* checked from a **stapled** OCSP response when the server sends one (see Trust
|
|
@@ -391,9 +468,29 @@ Not implemented, and not planned:
|
|
|
391
468
|
- **A public-suffix list for cookies.** Only the "no dot in the domain" guard is implemented, so
|
|
392
469
|
`Domain=com` is refused but `Domain=co.uk` is not. Documented rather than faked.
|
|
393
470
|
- **IDNA.** Pass A-labels (punycode); a non-ASCII hostname is rejected with a message saying so.
|
|
394
|
-
- **`br` and `zstd`
|
|
395
|
-
|
|
396
|
-
|
|
471
|
+
- **`br` and `zstd` out of the box.** The runtime's `DecompressionStream` accepts gzip, deflate and
|
|
472
|
+
deflate-raw only — measured, not assumed. Neither is *unreachable*, though: register a decoder
|
|
473
|
+
with [`decoders`](#br-zstd-and-other-codings) and the coding is advertised and decoded. Nothing
|
|
474
|
+
ships built in, because the only way to get Brotli here is WebAssembly, and a 208 KB binary blob
|
|
475
|
+
would cost this package both its zero dependencies and its ability to be imported without a
|
|
476
|
+
bundler. Bringing your own makes that cost, and that supply chain, yours and visible.
|
|
477
|
+
|
|
478
|
+
Leaving it off is safe rather than lossy: content negotiation means a server never sends what was
|
|
479
|
+
not asked for, so a Brotli-serving origin simply returns gzip. What it costs is bandwidth — the
|
|
480
|
+
same page measured 290 KB as gzip against 99 KB as `br` — and bandwidth is not what this platform
|
|
481
|
+
bills. Measured on the edge, the trade runs the wrong way: WASM Brotli decodes at about
|
|
482
|
+
**1.9x** the CPU of the runtime's native inflate, and even on the page with the largest wire
|
|
483
|
+
saving in a 14-site survey, the 186 KB saved bought back ~1.2 ms while the extra decoding cost
|
|
484
|
+
several times that. Harder compression makes it worse, not better — brotli quality 11, which is
|
|
485
|
+
what a CDN serves from cache, is 16% smaller on the wire than quality 5 and **46% more expensive
|
|
486
|
+
to decode**, because decompression work scales with the OUTPUT bytes and a denser encoding means
|
|
487
|
+
more work per byte produced. The reason to turn `br` on is matching a browser's
|
|
488
|
+
`Accept-Encoding`, not saving CPU.
|
|
489
|
+
- **Streaming request bodies.** A request body is read fully into memory before the request is
|
|
490
|
+
sent, because the framing has to be declared in a `Content-Length` this client can stand behind
|
|
491
|
+
and because a body may have to be replayed on a redirect. Fine for the JSON an SDK sends; wrong
|
|
492
|
+
for a large upload, and it means `duplex: 'half'` streaming uploads are not supported. Response
|
|
493
|
+
bodies are streamed throughout and are never buffered on your behalf.
|
|
397
494
|
- **HTTP/3.** ALPN offers `h2` and `http/1.1` (see [HTTP/2](#http2--access-not-speed)); it does
|
|
398
495
|
not offer `h3`, which is QUIC over UDP and unreachable from a runtime that exposes only raw TCP.
|
|
399
496
|
A server selecting anything the client did not offer fails closed — there is no fallback-and-retry
|
|
@@ -405,8 +502,11 @@ Not implemented, and not planned:
|
|
|
405
502
|
**Sockets cannot cross request contexts.** The pool is per-`Client` and per-invocation by design;
|
|
406
503
|
there is no cross-request connection cache, because the runtime does not permit one.
|
|
407
504
|
|
|
408
|
-
**Concurrency.** The
|
|
409
|
-
|
|
505
|
+
**Concurrency.** The limit is six connections **per Worker invocation** simultaneously awaiting
|
|
506
|
+
response headers — not per Worker and not per account, so separate requests to your Worker each get
|
|
507
|
+
their own six. A connection stops occupying a slot the moment its response head arrives, so the
|
|
508
|
+
limit bounds how many handshakes can be in flight at once, not how many bodies can be downloading.
|
|
509
|
+
A crawler wanting more parallelism inside one invocation must pipeline within that limit.
|
|
410
510
|
|
|
411
511
|
## Cost on a live Worker
|
|
412
512
|
|
|
@@ -570,8 +670,22 @@ sustained average rather than trusting either the documented number or the burst
|
|
|
570
670
|
|
|
571
671
|
WebCrypto (X25519, ECDH P-256/384/521, ECDSA, RSA-PSS, RSASSA-PKCS1, HKDF, HMAC, AES-GCM),
|
|
572
672
|
WHATWG Streams including BYOB readers, `TextEncoder`/`TextDecoder`, `DecompressionStream`, `URL`,
|
|
573
|
-
`Headers`, `Request`, `Response`, `AbortSignal`, `btoa`.
|
|
574
|
-
|
|
673
|
+
`Headers`, `Request`, `Response`, `AbortSignal`, `btoa`. The only runtime-specific piece is the
|
|
674
|
+
`connect` function you supply.
|
|
675
|
+
|
|
676
|
+
| Runtime | Offline suite | Notes |
|
|
677
|
+
|---|---|---|
|
|
678
|
+
| Node 22, 24 | **1132 / 1132** | what CI gates on |
|
|
679
|
+
| Node 20 | not supported | `TextDecoder` treats `iso-8859-1` as true ISO-8859-1 instead of aliasing it to windows-1252 as WHATWG requires, so bodies in that charset decode differently. Left maintenance April 2026 |
|
|
680
|
+
| workerd | live edge suite passes | the target runtime; exercised end to end by the scheduled edge job rather than by the offline suite |
|
|
681
|
+
| Deno 2.9 | 999 / 1001 | both failures are in the TLS 1.2 test server's secp521r1 path, not in the package; WebCrypto ECDSA and ECDH on P-521 both work standalone under Deno. Unresolved, so support is not claimed |
|
|
682
|
+
| Bun 1.3 | 1079 / 1082 | a module-resolution difference in one repo-hygiene test, one timing-sensitive deadline test, and the same TLS 1.2 suite test. Unresolved, so support is not claimed |
|
|
683
|
+
|
|
684
|
+
Node 22 and workerd are the supported pair. Deno and Bun very nearly work and are not tested in CI
|
|
685
|
+
— running the suite there is a good first contribution.
|
|
686
|
+
|
|
687
|
+
CI runs every version in `engines`, and a repo-hygiene test fails if the two ever disagree — an
|
|
688
|
+
untested support floor is a claim, not a fact.
|
|
575
689
|
|
|
576
690
|
## Testing
|
|
577
691
|
|
|
@@ -596,6 +710,26 @@ against independently written test servers — the 1.2 server is built on `node:
|
|
|
596
710
|
on this package's own primitives, so a client bug cannot be cancelled out by the same bug on the
|
|
597
711
|
server side.
|
|
598
712
|
|
|
713
|
+
Every parser that consumes bytes a peer controls — X.509, OCSP, HTTP/1.1 heads, chunked bodies,
|
|
714
|
+
HTTP/2 frames, HPACK — is fuzzed against one property: **any input either parses or throws a
|
|
715
|
+
`TunnelFetchError`.** An untyped throw, a `TypeError` from a missing null check or a `RangeError`
|
|
716
|
+
from a bad offset, is a finding: it means a check is missing and every caller relying on the typed
|
|
717
|
+
contract to fail closed will not catch it.
|
|
718
|
+
|
|
719
|
+
The fuzzer is seeded and dependency-free, so a failure prints the seed, the iteration and the case
|
|
720
|
+
in base64 — reproducible exactly, which a fuzzer whose failures cannot be replayed is not. Targets
|
|
721
|
+
are auto-discovered from `test/fuzz/targets/`; adding one is dropping a file in. The suite also
|
|
722
|
+
fuzzes *itself*: two synthetic targets prove the engine reports an untyped throw and does not report
|
|
723
|
+
a typed one, because a green fuzz run otherwise only proves the fuzzer never looked.
|
|
724
|
+
|
|
725
|
+
```bash
|
|
726
|
+
FUZZ_ITERATIONS=1000000 node --test test/fuzz/fuzz.test.js # a soak
|
|
727
|
+
FUZZ_SEED=12345 node --test test/fuzz/fuzz.test.js # a different corner
|
|
728
|
+
```
|
|
729
|
+
|
|
730
|
+
CI runs a fixed seed on every commit, as a gate; the scheduled workflow runs three million
|
|
731
|
+
iterations with the run id as the seed, which is the half that searches new ground.
|
|
732
|
+
|
|
599
733
|
`probe/` holds a reproducible capability probe that emits machine-readable JSON, and
|
|
600
734
|
`probe/results/` the measurements this design rests on. `live/` is the edge interop rig.
|
|
601
735
|
|
package/README.zh-CN.md
CHANGED
|
@@ -2,10 +2,17 @@
|
|
|
2
2
|
|
|
3
3
|
[English](README.md) · [简体中文](README.zh-CN.md)
|
|
4
4
|
|
|
5
|
+
[](https://github.com/latentharbor/tunnelfetch/actions/workflows/ci.yml)
|
|
6
|
+
[](https://www.npmjs.com/package/tunnelfetch)
|
|
7
|
+
[](LICENSE)
|
|
8
|
+
|
|
9
|
+
|
|
5
10
|
一个 `fetch` 形态的 HTTP 客户端,能在只暴露原始 TCP 的运行时上——主要是 Cloudflare Workers (`workerd`)——通过 HTTP CONNECT、HTTPS 或 SOCKS5 代理发出请求。
|
|
6
11
|
|
|
7
12
|
零依赖。ESM。无构建步骤。`src/` 中没有任何 `node:` 导入。
|
|
8
13
|
|
|
14
|
+
> **成熟度。** 这是一个新实现,不是久经沙场的实现。它在用户态实现了 TLS 1.2/1.3 与证书校验——在这个领域,好的测试是必要条件,但不是充分条件。它有 1132 个离线测试、RFC 向量、逐字节分片、真实边缘互操作测试、对每一个面向对端的解析器做种子化 fuzzing,行覆盖率 95%;但它**没有**经过外部安全审计。请把它当作一个值得一试的高质量实现,而不是一个已被生产验证的东西。发现任何问题都欢迎报告——见 [SECURITY.md](SECURITY.md)。
|
|
15
|
+
|
|
9
16
|
```js
|
|
10
17
|
import { Client } from 'tunnelfetch';
|
|
11
18
|
import { connect } from 'cloudflare:sockets';
|
|
@@ -165,6 +172,53 @@ for await (const chunk of res.body) {
|
|
|
165
172
|
|
|
166
173
|
中途放弃一条流,连接绝不会回到池里:它的位置是未知的,复用会把上一个响应的残余拼接到下一个请求上。
|
|
167
174
|
|
|
175
|
+
### `br`、`zstd` 与其它编码
|
|
176
|
+
|
|
177
|
+
`gzip` 和 `deflate` 是内置的,因为运行时原生就能解。其余都是可插拔的:给 `decoders` 传一个"每种编码一个函数"的表,该编码就会被追加进 `Accept-Encoding`,并被用于解码匹配的响应。注册这个动作本身,正是"声明"变得诚实的前提——请求一个自己读不了的编码,会把每一个这样的响应变成乱码,所以两者绑在一起、不可能漂移。
|
|
178
|
+
|
|
179
|
+
```js
|
|
180
|
+
import { Client } from 'tunnelfetch';
|
|
181
|
+
import { connect } from 'cloudflare:sockets';
|
|
182
|
+
import { BrotliDecStream, BrotliStreamResultCode, initSync } from 'brotli-dec-wasm/web';
|
|
183
|
+
import wasm from 'brotli-dec-wasm/web/bg.wasm';
|
|
184
|
+
|
|
185
|
+
// 放在模块作用域,实例化就落进 isolate 启动阶段,而这个运行时不对启动计费。
|
|
186
|
+
// 边缘实测:这一步的成本测不出来(带它启动 12 ms,不带也是 12 ms)。
|
|
187
|
+
initSync({ module: wasm });
|
|
188
|
+
|
|
189
|
+
const brotli = (stream) => {
|
|
190
|
+
const dec = new BrotliDecStream();
|
|
191
|
+
return stream.pipeThrough(new TransformStream({
|
|
192
|
+
transform(chunk, c) {
|
|
193
|
+
let r = dec.dec(chunk, 1 << 20);
|
|
194
|
+
if (r.buf.length) c.enqueue(r.buf);
|
|
195
|
+
while (r.code === BrotliStreamResultCode.NeedsMoreOutput) {
|
|
196
|
+
r = dec.dec(new Uint8Array(0), 1 << 20);
|
|
197
|
+
if (r.buf.length) c.enqueue(r.buf);
|
|
198
|
+
}
|
|
199
|
+
},
|
|
200
|
+
}));
|
|
201
|
+
};
|
|
202
|
+
|
|
203
|
+
const client = new Client({ connect, proxy, decoders: { br: brotli } });
|
|
204
|
+
// 此后会发送 `Accept-Encoding: gzip, deflate, br`,并解码 `Content-Encoding: br`。
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
顺序是内置项之后按注册顺序排,所以 `{ br, zstd }` 得到的正是 Chrome 发的 `gzip, deflate, br, zstd`——而这才是做这件事的真正理由。这个客户端默认呈现 curl 的 TLS 与 HTTP/2 指纹,而 `gzip, deflate` 正是 curl 发的,所以默认状态本就是一致的。它不再一致,恰恰是从你把握手打扮成浏览器、却把这个头留在原地的那一刻开始。
|
|
208
|
+
|
|
209
|
+
它不是一项省钱优化。边缘实测,把同一个 256 KB 页面解成同样的字节:
|
|
210
|
+
|
|
211
|
+
| 实现 | 算法 | ms/MB |
|
|
212
|
+
|---|---|---|
|
|
213
|
+
| `DecompressionStream`——运行时自己的 C++ | inflate | **2.5** |
|
|
214
|
+
| WASM brotli(`brotli-dec-wasm`) | brotli | 4.7 |
|
|
215
|
+
| JS inflate(`fflate` / `pako`) | inflate | 7.5 / 8.2 |
|
|
216
|
+
| JS brotli(`brotli`) | brotli | 19.7 |
|
|
217
|
+
|
|
218
|
+
由此得到两条结论,在 Worker 里任何地方考虑用 WebAssembly 之前都值得知道。**同一算法下 WASM 比 JavaScript 快约 4 倍**(brotli:4.7 对 19.7)——所以某个编码若没有原生实现,WASM 是补上它的正确方式。**同一算法下原生又比 JavaScript 快约 3 倍**(inflate:2.5 对 7.5–8.2)——所以只要已经有原生路径,用户态怎么写都赢不了。这正是 `gzip` 和 `deflate` 不可覆盖的原因:替换它们只可能更慢,而静默地这么做,恰恰是这个包在别处一律拒绝的那种悄悄降级。
|
|
219
|
+
|
|
220
|
+
brotli 本身落在原生 inflate 的 1.9 倍。这个差价就是这个编码的成本,而它省下的线上字节买不回来——见[这个包做不到什么](#这个包做不到什么为什么)。解码器名字会按 HTTP token 校验;解码器抛错时响应体 fail closed,而不是被截断;未注册的编码依然会被拒绝。
|
|
221
|
+
|
|
168
222
|
### HTTP/2 — 要的是访问,不是速度
|
|
169
223
|
|
|
170
224
|
客户端默认在 ALPN 中同时报出 `h2` 与 `http/1.1`,服务器选中哪个就说哪个。没有单独的 API:
|
|
@@ -295,19 +349,22 @@ CertificateError [CERT_PIN_MISMATCH]: no certificate in the chain matches any co
|
|
|
295
349
|
- **TLS 1.0 / 1.1。** RC4 已被攻破,那里剩下的全是 CBC。浏览器自 2020 年起就拒绝这两个版本。注意其中的区别:*同时也*支持 1.0/1.1 的服务器没有问题,我们会与它协商出 1.2 或 1.3;够不着的只有*除此之外什么都不支持*的服务器。
|
|
296
350
|
- **RSA 密钥传输。** 没有前向保密。
|
|
297
351
|
- **ChaCha20-Poly1305。** 加上它换不来任何东西:服务器只能从我们报出的套件里选,TLS 1.3 强制要求 AES-128-GCM,而 AES-GCM 在 TLS 1.2 的实际部署中无处不在。WebCrypto 没有 ChaCha20,要提供它就得引入 `node:crypto` 依赖,兼容性收益却是零。
|
|
298
|
-
-
|
|
352
|
+
- **客户端证书(mTLS)、0-RTT、重协商。** `HelloRequest` 会被拒绝而不是照做。0-RTT 是决定而非遗漏:early data 可被重放,提供它等于让捕获了 POST 的攻击者可以重放它。(会话恢复本身**已经实现**——ticket 按池键保存,用 `psk_dhe_ke` 提供。)
|
|
299
353
|
- **吊销信息的主动获取(下载 CRL、查询 OCSP 应答者)。** 两者都要在握手中途经代理多跑网络往返,而且 OCSP 查询会把你访问的源站告诉 CA。吊销状态**会**从服务器装订(staple)的 OCSP 响应中校验(见上文 Trust 一节);没有实现、也不打算实现的,是替服务器去取它没有装订的东西。
|
|
300
354
|
- **证书策略处理** (`policyConstraints`、`inhibitAnyPolicy`)。它们永远是 critical 的,所以一旦出现就直接拒绝,而不是被错误地校验过去。
|
|
301
355
|
- **dNSName 与 iPAddress 之外的名称约束。** 标为 *critical* 且指名不支持类型的约束扩展会被拒绝;非 critical 的则按 RFC 5280 允许的那样忽略。
|
|
302
356
|
- **cookie 的 public suffix list。** 只实现了“域名里没有点”这一道防护,所以 `Domain=com` 会被拒绝,`Domain=co.uk` 不会。如实写明,而不是伪造。
|
|
303
357
|
- **IDNA。** 请传 A-label (punycode);非 ASCII 主机名会被拒绝,并附上说明原因的报错。
|
|
304
|
-
-
|
|
358
|
+
- **开箱即用的 `br` 与 `zstd`。** 运行时的 `DecompressionStream` 只接受 gzip、deflate 和 deflate-raw——这是实测的,不是假设的。但这两种编码并非够不着:用 [`decoders`](#br-zstd-与其它编码) 注册一个解码器,该编码就会被声明并被解码。包里不内置任何一个,是因为在这个运行时上拿到 Brotli 的唯一途径是 WebAssembly,而一个 208 KB 的二进制块会同时让这个包失去零依赖、以及"不经打包器即可导入"的可移植性。自带解码器,等于把那份成本和那条供应链变成你自己的、并且看得见的。
|
|
359
|
+
|
|
360
|
+
不开它是安全的,而不是有损的:内容协商决定了服务器绝不会发送没被请求的编码,所以提供 Brotli 的源站只会返回 gzip。代价是带宽——同一个页面 gzip 是 290 KB,`br` 是 99 KB——而带宽恰恰不是这个平台计费的东西。边缘实测下来,这笔交易是反着的:WASM Brotli 的 CPU 约为运行时原生 inflate 的 **1.9 倍**;即便在 14 个站点里线上字节省得最多的那一个上,省下的 186 KB 只换回约 1.2 ms,而多出来的解压是它的好几倍。而且压得越狠越亏,不是越省——brotli quality 11(CDN 从缓存里发的就是它)比 quality 5 线上小 16%,解码却贵 **46%**,因为解压的工作量跟着**输出**字节走,而更密的编码意味着每产出一个字节要做更多活。开 `br` 的理由是让 `Accept-Encoding` 与浏览器一致,不是省 CPU。
|
|
361
|
+
- **流式请求体。** 请求体会先完整读入内存再发出:一是框定长度需要一个这个客户端敢于担保的 `Content-Length`,二是遇到重定向时可能需要重放。对 SDK 发的那点 JSON 完全够用;对大文件上传就不合适,也意味着不支持 `duplex: 'half'` 的流式上传。响应体则全程流式,任何时候都不会替你缓冲。
|
|
305
362
|
- **HTTP/3。** ALPN 报出的是 `h2` 与 `http/1.1`(见 [HTTP/2](#http2--要的是访问不是速度));不报 `h3`——那是跑在 UDP 上的 QUIC,从一个只暴露原始 TCP 的运行时根本够不着。服务器若选中客户端未曾报出的协议,一律 fail closed——任何一层都不存在回退重试。
|
|
306
363
|
- **服务器推送、HTTP/2 优先级与 h2c。** 推送在我们的 SETTINGS 里就是关闭的,收到 `PUSH_PROMISE` 即为连接错误;RFC 9113 的优先级机制已被废弃,PRIORITY 帧一律忽略;h2 只跑在 ALPN 协商出的 TLS 之上,绝不以“prior knowledge”走明文。
|
|
307
364
|
|
|
308
365
|
**socket 不能跨请求上下文。** 连接池按设计随 `Client`、随单次调用存在;不存在跨请求的连接缓存,因为运行时不允许有。
|
|
309
366
|
|
|
310
|
-
**并发。**
|
|
367
|
+
**并发。** 限制是**每次 Worker 调用**最多六条连接同时处于等待响应头的状态——不是每个 Worker,也不是每个账号,所以打到你 Worker 的不同请求各有各的六条。连接在响应头到达的那一刻就腾出槽位,所以它约束的是「同时能有多少次握手在飞」,而不是「同时能下载多少个 body」。想在一次调用内跑更高并行度的爬虫,必须在这个限额之内做流水线。
|
|
311
368
|
|
|
312
369
|
## 线上 Worker 的开销
|
|
313
370
|
|
|
@@ -440,7 +497,19 @@ inflate 本身(约 2 ms/MB)加上把 body 物化成 JS 字符串(约 1.7 m
|
|
|
440
497
|
|
|
441
498
|
## 运行时要求
|
|
442
499
|
|
|
443
|
-
WebCrypto (X25519、ECDH P-256/384/521、ECDSA、RSA-PSS、RSASSA-PKCS1、HKDF、HMAC、AES-GCM)、含 BYOB reader 的 WHATWG Streams、`TextEncoder`/`TextDecoder`、`DecompressionStream`、`URL`、`Headers`、`Request`、`Response`、`AbortSignal`、`btoa`。这些在 workerd
|
|
500
|
+
WebCrypto (X25519、ECDH P-256/384/521、ECDSA、RSA-PSS、RSASSA-PKCS1、HKDF、HMAC、AES-GCM)、含 BYOB reader 的 WHATWG Streams、`TextEncoder`/`TextDecoder`、`DecompressionStream`、`URL`、`Headers`、`Request`、`Response`、`AbortSignal`、`btoa`。这些在 workerd 与 Node ≥ 22 上全部具备。唯一与具体运行时绑定的部分,是你自己提供的 `connect` 函数。
|
|
501
|
+
|
|
502
|
+
| 运行时 | 离线测试 | 说明 |
|
|
503
|
+
|---|---|---|
|
|
504
|
+
| Node 22、24 | **1132 / 1132** | CI 把关的组合 |
|
|
505
|
+
| Node 20 | 不支持 | `TextDecoder` 把 `iso-8859-1` 当作真正的 ISO-8859-1,而没有按 WHATWG 要求别名到 windows-1252,该字符集的响应体解码结果不同。已于 2026 年 4 月结束维护 |
|
|
506
|
+
| workerd | live 边缘测试全过 | 目标运行时;由定时边缘任务端到端验证,不在离线套件里 |
|
|
507
|
+
| Deno 2.9 | 999 / 1001 | 两个失败都落在 TLS 1.2 测试服务器的 secp521r1 路径上,不在包内;Deno 的 WebCrypto ECDSA 与 ECDH 在 P-521 上单独测试都正常。原因未定位,因此不宣称支持 |
|
|
508
|
+
| Bun 1.3 | 1079 / 1082 | 一个 repo-hygiene 测试的模块解析差异、一个对时序敏感的 deadline 测试,以及同一个 TLS 1.2 套件测试。原因未定位,因此不宣称支持 |
|
|
509
|
+
|
|
510
|
+
受支持的组合是 Node 22 与 workerd。Deno 和 Bun 已经非常接近可用,但没有进 CI——把这套测试在那两个运行时上跑通,是一个很好的首次贡献。
|
|
511
|
+
|
|
512
|
+
CI 会跑 `engines` 里声明的每一个版本,并且有一个 repo-hygiene 测试在两者不一致时直接失败——一个没人跑过的支持下限,是声称,不是事实。
|
|
444
513
|
|
|
445
514
|
## 测试
|
|
446
515
|
|
|
@@ -455,6 +524,17 @@ npm run test:live # explicit; needs TUNNELFETCH_PROXY in the environment
|
|
|
455
524
|
|
|
456
525
|
TLS 密钥调度与记录层逐字节钉死在 **RFC 8448** "Example Handshake Traces for TLS 1.3" 上:AEAD 复现出 RFC 里一模一样的密文记录,记录层把 RFC 的完整线上字节在两个方向上原样重放。两个 TLS 驱动都对着独立编写的测试服务器测试——1.2 的服务器建在 `node:crypto` 上,而不是本包自己的原语上,这样客户端的 bug 就不可能被服务器端的同一个 bug 抵消掉。
|
|
457
526
|
|
|
527
|
+
每一个消费对端字节的解析器——X.509、OCSP、HTTP/1.1 响应头、分块体、HTTP/2 帧、HPACK——都会被 fuzz,断言的属性只有一条:**任意输入,要么解析成功,要么抛出 `TunnelFetchError`。** 抛出未类型化的错误(少了空值检查的 `TypeError`、越界的 `RangeError`)就是一个发现:它意味着某处检查缺失,而所有依赖类型化契约来 fail closed 的调用方都接不住它。
|
|
528
|
+
|
|
529
|
+
fuzzer 自带种子、零依赖,失败时会打印种子、迭代序号和 base64 的输入——可以精确复现,而无法复现失败的 fuzzer 算不上 fuzzer。目标从 `test/fuzz/targets/` 自动发现,新增一个就是放一个文件进去。这套测试还会 fuzz **它自己**:两个合成目标分别证明引擎能报出未类型化的抛出、且不会误报类型化的抛出——否则一次全绿的 fuzz 只能证明 fuzzer 从来没睁眼。
|
|
530
|
+
|
|
531
|
+
```bash
|
|
532
|
+
FUZZ_ITERATIONS=1000000 node --test test/fuzz/fuzz.test.js # 长跑
|
|
533
|
+
FUZZ_SEED=12345 node --test test/fuzz/fuzz.test.js # 换一个角落
|
|
534
|
+
```
|
|
535
|
+
|
|
536
|
+
CI 每次提交跑固定种子,作为门禁;定时任务跑三百万次迭代、用 run id 作种子,那才是真正在搜索新地面的那一半。
|
|
537
|
+
|
|
458
538
|
`probe/` 存放一个可复现的能力探测,输出机器可读的 JSON;`probe/results/` 存放本设计所依据的测量结果。`live/` 是边缘互操作实测环境。
|
|
459
539
|
|
|
460
540
|
凭据只从环境读取。live 套件在未配置时会大声失败而不是跳过:一个意思是“我们没检查”的绿勾,比红叉更糟。
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "tunnelfetch",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.1.0",
|
|
4
4
|
"description": "A fetch-shaped HTTP client that can route through HTTP CONNECT / HTTPS / SOCKS5 proxies on runtimes with only raw TCP, such as Cloudflare Workers. Implements TLS in userland because the runtime cannot verify a tunnelled peer.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"fetch",
|
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
"type": "module",
|
|
26
26
|
"sideEffects": false,
|
|
27
27
|
"engines": {
|
|
28
|
-
"node": ">=
|
|
28
|
+
"node": ">=22"
|
|
29
29
|
},
|
|
30
30
|
"exports": {
|
|
31
31
|
".": {
|
package/src/client/decode.js
CHANGED
|
@@ -6,13 +6,72 @@
|
|
|
6
6
|
import { HttpError, codes } from '../errors.js';
|
|
7
7
|
|
|
8
8
|
/**
|
|
9
|
-
*
|
|
9
|
+
* What the request layer advertises with no extra decoders registered. The target runtime's
|
|
10
10
|
* DecompressionStream supports ONLY gzip / deflate / deflate-raw (verified empirically);
|
|
11
11
|
* advertising `br` or `zstd` would invite bytes we can never decode, turning every response
|
|
12
12
|
* from a brotli-preferring CDN into garbage. Keep this list and decodeBody in lockstep.
|
|
13
|
+
*
|
|
14
|
+
* It is also exactly what curl sends, which matters because this client presents curl's TLS and
|
|
15
|
+
* HTTP/2 fingerprints by default: a browser-shaped handshake paired with a curl-shaped
|
|
16
|
+
* Accept-Encoding is a mismatch a bot detector can read straight off the wire.
|
|
13
17
|
*/
|
|
14
18
|
export const ACCEPT_ENCODING = 'gzip, deflate';
|
|
15
19
|
|
|
20
|
+
/** Codings the runtime decodes itself. Not overridable — see acceptEncodingFor. */
|
|
21
|
+
const BUILT_IN = new Set(['gzip', 'x-gzip', 'deflate', 'identity']);
|
|
22
|
+
|
|
23
|
+
/** An HTTP token, which is what a content-coding name must be (RFC 9110 §5.6.2). */
|
|
24
|
+
const TOKEN = /^[!#$%&'*+\-.^_`|~0-9a-z]+$/;
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* The Accept-Encoding to send given the caller's extra decoders.
|
|
28
|
+
*
|
|
29
|
+
* Registering a decoder is what makes advertising its coding honest — the two must move together
|
|
30
|
+
* or the client asks for bytes it cannot read. Order is registration order after the built-ins,
|
|
31
|
+
* so a caller matching a browser can produce exactly `gzip, deflate, br, zstd`.
|
|
32
|
+
*
|
|
33
|
+
* @param {Record<string, BodyDecoder> | null | undefined} decoders
|
|
34
|
+
* @returns {string}
|
|
35
|
+
*/
|
|
36
|
+
export function acceptEncodingFor(decoders) {
|
|
37
|
+
if (!decoders) return ACCEPT_ENCODING;
|
|
38
|
+
const extra = [];
|
|
39
|
+
for (const name of Object.keys(decoders)) {
|
|
40
|
+
const coding = name.toLowerCase();
|
|
41
|
+
// A name is interpolated into a request header, so anything that is not a bare token could
|
|
42
|
+
// inject a second header field. Reject at registration rather than at send time.
|
|
43
|
+
if (!TOKEN.test(coding)) {
|
|
44
|
+
throw new HttpError(
|
|
45
|
+
codes.HTTP_CONTENT_ENCODING,
|
|
46
|
+
`"${name}" is not a valid content-coding name; expected an HTTP token`,
|
|
47
|
+
{ coding: name },
|
|
48
|
+
);
|
|
49
|
+
}
|
|
50
|
+
// A built-in must not be overridable. The runtime decompresses gzip and deflate in its own
|
|
51
|
+
// native code, measured at roughly a third the CPU of the fastest JavaScript implementations,
|
|
52
|
+
// so accepting a replacement would silently trade that away — and the two code paths disagreed
|
|
53
|
+
// about it: this function treated `gzip` as already covered while decodeBody would have used
|
|
54
|
+
// the caller's function. Refusing here makes both paths say the same thing.
|
|
55
|
+
if (BUILT_IN.has(coding)) {
|
|
56
|
+
throw new HttpError(
|
|
57
|
+
codes.HTTP_CONTENT_ENCODING,
|
|
58
|
+
`"${coding}" is decoded by the runtime natively and cannot be replaced; remove it from ` +
|
|
59
|
+
'`decoders`. Only codings this runtime has no decoder for can be supplied.',
|
|
60
|
+
{ coding },
|
|
61
|
+
);
|
|
62
|
+
}
|
|
63
|
+
if (typeof decoders[name] !== 'function') {
|
|
64
|
+
throw new HttpError(
|
|
65
|
+
codes.HTTP_CONTENT_ENCODING,
|
|
66
|
+
`the decoder registered for "${coding}" is not a function`,
|
|
67
|
+
{ coding },
|
|
68
|
+
);
|
|
69
|
+
}
|
|
70
|
+
extra.push(coding);
|
|
71
|
+
}
|
|
72
|
+
return extra.length ? `${ACCEPT_ENCODING}, ${extra.join(', ')}` : ACCEPT_ENCODING;
|
|
73
|
+
}
|
|
74
|
+
|
|
16
75
|
/** Codings that exist and are real but that this runtime cannot decompress. */
|
|
17
76
|
const KNOWN_UNSUPPORTED = new Set(['br', 'zstd', 'compress', 'x-compress']);
|
|
18
77
|
|
|
@@ -192,13 +251,29 @@ function firstBytes(chunks, i) {
|
|
|
192
251
|
/**
|
|
193
252
|
* Undo the response's Content-Encoding.
|
|
194
253
|
*
|
|
254
|
+
/**
|
|
255
|
+
* A caller-supplied content decoder: raw coded bytes in, decoded bytes out. Streaming, so a body
|
|
256
|
+
* is never fully buffered on this client's behalf.
|
|
257
|
+
* @typedef {(stream: ReadableStream<Uint8Array>) => ReadableStream<Uint8Array>} BodyDecoder
|
|
258
|
+
*/
|
|
259
|
+
|
|
260
|
+
/**
|
|
195
261
|
* @param {ReadableStream<Uint8Array>} stream the raw body
|
|
196
262
|
* @param {string|null|undefined} contentEncoding the Content-Encoding header value; a
|
|
197
263
|
* comma-separated list names codings in the order the SERVER applied them, so decoding
|
|
198
264
|
* applies them in reverse.
|
|
265
|
+
* @param {Record<string, BodyDecoder> | null} [decoders] caller-supplied codings
|
|
199
266
|
* @returns {ReadableStream<Uint8Array>} decoded bytes
|
|
200
267
|
*/
|
|
201
|
-
export function decodeBody(stream, contentEncoding) {
|
|
268
|
+
export function decodeBody(stream, contentEncoding, decoders = null) {
|
|
269
|
+
/** Look a coding up among the caller's decoders, case-insensitively as the header is. */
|
|
270
|
+
const custom = (coding) => {
|
|
271
|
+
if (!decoders) return null;
|
|
272
|
+
for (const name of Object.keys(decoders)) {
|
|
273
|
+
if (name.toLowerCase() === coding) return decoders[name];
|
|
274
|
+
}
|
|
275
|
+
return null;
|
|
276
|
+
};
|
|
202
277
|
const codings = (contentEncoding ?? '')
|
|
203
278
|
.split(',')
|
|
204
279
|
.map((c) => c.trim().toLowerCase())
|
|
@@ -209,12 +284,17 @@ export function decodeBody(stream, contentEncoding) {
|
|
|
209
284
|
if (coding === 'gzip' || coding === 'x-gzip' || coding === 'deflate' || coding === 'identity') {
|
|
210
285
|
continue;
|
|
211
286
|
}
|
|
287
|
+
// A registered decoder is the caller taking responsibility for this coding; it outranks the
|
|
288
|
+
// built-in refusal, which only ever said "this runtime cannot".
|
|
289
|
+
if (custom(coding)) continue;
|
|
212
290
|
if (KNOWN_UNSUPPORTED.has(coding)) {
|
|
213
291
|
throw new HttpError(
|
|
214
292
|
codes.HTTP_CONTENT_ENCODING,
|
|
215
|
-
`Content-Encoding "${coding}" is not decodable:
|
|
216
|
-
'
|
|
217
|
-
|
|
293
|
+
`Content-Encoding "${coding}" is not decodable: no decoder is registered for it and this ` +
|
|
294
|
+
"runtime's DecompressionStream " +
|
|
295
|
+
'supports only gzip/deflate/deflate-raw, which is why this client does not advertise ' +
|
|
296
|
+
`"${coding}" in Accept-Encoding — the server should not have sent it. Pass ` +
|
|
297
|
+
`\`decoders: { '${coding}': fn }\` to supply one.`,
|
|
218
298
|
{ coding },
|
|
219
299
|
);
|
|
220
300
|
}
|
|
@@ -226,6 +306,24 @@ export function decodeBody(stream, contentEncoding) {
|
|
|
226
306
|
for (let i = codings.length - 1; i >= 0; i--) {
|
|
227
307
|
const coding = codings[i];
|
|
228
308
|
if (coding === 'identity') continue; // no-op by definition
|
|
309
|
+
// Built-ins take the native path unconditionally; acceptEncodingFor refuses to register one,
|
|
310
|
+
// and this is the same rule stated where the decoding actually happens, so neither entry
|
|
311
|
+
// point can be reached with the other's assumption.
|
|
312
|
+
const fn = BUILT_IN.has(coding) ? null : custom(coding);
|
|
313
|
+
if (fn) {
|
|
314
|
+
const staged = fn(out);
|
|
315
|
+
// A decoder that returns something unreadable would surface far downstream as a confusing
|
|
316
|
+
// stream error; name it here, where the caller can see which coding misbehaved.
|
|
317
|
+
if (!staged || typeof staged.getReader !== 'function') {
|
|
318
|
+
throw new HttpError(
|
|
319
|
+
codes.HTTP_CONTENT_ENCODING,
|
|
320
|
+
`the decoder for "${coding}" did not return a ReadableStream`,
|
|
321
|
+
{ coding },
|
|
322
|
+
);
|
|
323
|
+
}
|
|
324
|
+
out = staged;
|
|
325
|
+
continue;
|
|
326
|
+
}
|
|
229
327
|
out = decompressionStage(out, coding === 'x-gzip' ? 'gzip' : coding);
|
|
230
328
|
}
|
|
231
329
|
return out;
|
package/src/client.js
CHANGED
|
@@ -14,7 +14,7 @@ import { ConfigError, HttpError, TunnelFetchError, codes } from './errors.js';
|
|
|
14
14
|
import { ByteReader, ByteWriter, UnexpectedEofError, concat, utf8 } from './util/bytes.js';
|
|
15
15
|
import { serializeRequestHead } from './http1/request.js';
|
|
16
16
|
import { bodyFraming, readResponseBody, readResponseHead } from './http1/response.js';
|
|
17
|
-
import {
|
|
17
|
+
import { acceptEncodingFor, decodeBody } from './client/decode.js';
|
|
18
18
|
import { CookieJar } from './client/cookies.js';
|
|
19
19
|
import { DEFAULT_MAX_REDIRECTS, nextRequest, shouldRedirect } from './client/redirect.js';
|
|
20
20
|
import { ConnectionPool, poolKey } from './pool.js';
|
|
@@ -71,8 +71,16 @@ const NULL_BODY_STATUS = new Set([101, 204, 205, 304]);
|
|
|
71
71
|
* one across Clients or to persist it.
|
|
72
72
|
* @property {number} [maxRedirects] default 20.
|
|
73
73
|
* @property {number} [maxBodyBytes] enforced from Content-Length before a byte is read.
|
|
74
|
-
* @property {boolean} [decompress] gzip/deflate. Default true.
|
|
75
|
-
*
|
|
74
|
+
* @property {boolean} [decompress] gzip/deflate. Default true.
|
|
75
|
+
* @property {Record<string, import('./client/decode.js').BodyDecoder>} [decoders] extra
|
|
76
|
+
* content-codings this client can read, e.g. `{ br: (s) => ... }`. Registering one is what
|
|
77
|
+
* makes advertising it honest, so each name is appended to Accept-Encoding — a client that
|
|
78
|
+
* asked for a coding it cannot decode would turn every such response into garbage. `br` and
|
|
79
|
+
* `zstd` are not built in because the runtime's DecompressionStream has neither and this
|
|
80
|
+
* package takes no dependencies; supply your own and the cost, and the supply chain, are
|
|
81
|
+
* yours and visible. Measured on the edge: WASM brotli decodes at about 2x native gzip, and
|
|
82
|
+
* the wire bytes it saves do not pay that back — see the README. The reason to turn it on is
|
|
83
|
+
* matching a browser's Accept-Encoding, not saving CPU.
|
|
76
84
|
* @property {boolean} [keepAlive] default true.
|
|
77
85
|
* @property {boolean} [http2] offer HTTP/2 via ALPN and speak it when the server selects it.
|
|
78
86
|
* Default true. The goal is ACCESS, not speed — some sites treat HTTP/1.1 as a bot signal — and
|
|
@@ -93,7 +101,7 @@ export class Client {
|
|
|
93
101
|
* @param {ClientOptions} [options]
|
|
94
102
|
*/
|
|
95
103
|
constructor(options = {}) {
|
|
96
|
-
this.options =
|
|
104
|
+
this.options = snapshotOptions(options);
|
|
97
105
|
this.pool = new ConnectionPool(options.pool);
|
|
98
106
|
// HTTP/2 connections are NOT pooled the way h1 is: one connection multiplexes many concurrent
|
|
99
107
|
// streams, so it is not checked out per request. It lives here, keyed exactly like the h1 pool,
|
|
@@ -118,6 +126,12 @@ export class Client {
|
|
|
118
126
|
typeof options.now === 'number' ? { now: () => options.now } : {},
|
|
119
127
|
);
|
|
120
128
|
this._closed = false;
|
|
129
|
+
// Response bodies that have not finished arriving. `fetch` resolves at the response HEAD, so a
|
|
130
|
+
// caller holding a Response may still be streaming over a connection this Client owns — and on
|
|
131
|
+
// HTTP/2 that connection is SHARED and torn down by close(), not checked out of the pool the
|
|
132
|
+
// way an h1 socket is. Anything that decides when a Client is done needs to see these.
|
|
133
|
+
/** @type {Set<Promise<void>>} */
|
|
134
|
+
this._inflight = new Set();
|
|
121
135
|
// Bound so a Client can be handed straight to an SDK expecting a bare function.
|
|
122
136
|
this.fetch = this.fetch.bind(this);
|
|
123
137
|
}
|
|
@@ -134,6 +148,19 @@ export class Client {
|
|
|
134
148
|
return performFetch(this, input, init);
|
|
135
149
|
}
|
|
136
150
|
|
|
151
|
+
/**
|
|
152
|
+
* Resolve once every response body handed out by this Client has finished, one way or another —
|
|
153
|
+
* read to the end, cancelled, or failed. Deliberately NOT folded into close(): close() is the
|
|
154
|
+
* forceful teardown, and a teardown that waits on the streams it is tearing down would hang on
|
|
155
|
+
* any body the caller abandoned. This is for callers that want the graceful order.
|
|
156
|
+
*
|
|
157
|
+
* Looping rather than a single Promise.all because a body settling can start another (a redirect
|
|
158
|
+
* drains its predecessor), and a set sampled once would miss the successor.
|
|
159
|
+
*/
|
|
160
|
+
async idle() {
|
|
161
|
+
while (this._inflight.size) await Promise.all([...this._inflight]);
|
|
162
|
+
}
|
|
163
|
+
|
|
137
164
|
/** Release every pooled socket and shared HTTP/2 connection. A Client that is finished must be
|
|
138
165
|
* closed or sockets leak for the isolate's lifetime. */
|
|
139
166
|
async close() {
|
|
@@ -156,11 +183,27 @@ export class Client {
|
|
|
156
183
|
export function createFetch(options = {}) {
|
|
157
184
|
return async function tunnelFetch(input, init) {
|
|
158
185
|
const client = new Client(options);
|
|
186
|
+
let response;
|
|
159
187
|
try {
|
|
160
|
-
|
|
161
|
-
}
|
|
188
|
+
response = await client.fetch(input, init);
|
|
189
|
+
} catch (err) {
|
|
162
190
|
await client.close();
|
|
191
|
+
throw err;
|
|
163
192
|
}
|
|
193
|
+
// NOT `finally { await client.close() }`. `fetch` resolves when the response HEAD arrives and
|
|
194
|
+
// the body is still on the wire, so closing here tore down the connection out from under the
|
|
195
|
+
// caller — invisibly on HTTP/1.1, where the socket is checked out of the pool and closeAll()
|
|
196
|
+
// could not reach it, but fatally on HTTP/2, where the connection is shared and close() ends
|
|
197
|
+
// every stream on it. `res.text()` then failed with HTTP2_PROTOCOL.
|
|
198
|
+
//
|
|
199
|
+
// Not awaited, or this would deadlock: the body only drains when the caller reads it, and the
|
|
200
|
+
// caller cannot read a Response that has not been returned yet.
|
|
201
|
+
//
|
|
202
|
+
// A caller who neither reads nor cancels the body would leave the Client open — the same leak
|
|
203
|
+
// as forgetting to close one — except that the idle deadline aborts a stalled body, which
|
|
204
|
+
// settles the completion and fires this anyway.
|
|
205
|
+
void client.idle().then(() => client.close().catch(() => {}));
|
|
206
|
+
return response;
|
|
164
207
|
};
|
|
165
208
|
}
|
|
166
209
|
|
|
@@ -422,6 +465,48 @@ function registerHttp2(client, key, conn) {
|
|
|
422
465
|
* @param {unknown} err
|
|
423
466
|
* @returns {boolean}
|
|
424
467
|
*/
|
|
468
|
+
/**
|
|
469
|
+
* Take a private, frozen copy of the security-relevant configuration.
|
|
470
|
+
*
|
|
471
|
+
* `{ ...options }` is a SHALLOW copy, so `client.options.trust` stayed the caller's own object: a
|
|
472
|
+
* caller could flip `revocation` or swap `pins` after construction and the next request would run
|
|
473
|
+
* under the new policy over a pool of connections verified under the old one. The pool key covers
|
|
474
|
+
* every field the verifier reads (see trustFingerprint), but a key computed from a mutable object
|
|
475
|
+
* is only as stable as the object.
|
|
476
|
+
*
|
|
477
|
+
* `proxy` is deliberately NOT frozen: openTunnel treats a frozen proxy as already normalised and
|
|
478
|
+
* skips parseProxy, so freezing a raw object here would bypass its validation entirely. Proxy
|
|
479
|
+
* identity is part of the pool key on its own, so a changed proxy gets a different key regardless.
|
|
480
|
+
*
|
|
481
|
+
* @param {ClientOptions} options
|
|
482
|
+
* @returns {Readonly<ClientOptions>}
|
|
483
|
+
*/
|
|
484
|
+
function snapshotOptions(options) {
|
|
485
|
+
const o = { ...options };
|
|
486
|
+
if (o.trust && typeof o.trust === 'object') {
|
|
487
|
+
const t = { ...o.trust };
|
|
488
|
+
if (Array.isArray(t.pins)) t.pins = Object.freeze([...t.pins]);
|
|
489
|
+
if (Array.isArray(t.anchors)) t.anchors = Object.freeze([...t.anchors]);
|
|
490
|
+
o.trust = Object.freeze(t);
|
|
491
|
+
}
|
|
492
|
+
if (o.tls && typeof o.tls === 'object') o.tls = Object.freeze({ ...o.tls });
|
|
493
|
+
if (o.timeouts && typeof o.timeouts === 'object') o.timeouts = Object.freeze({ ...o.timeouts });
|
|
494
|
+
return Object.freeze(o);
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
/**
|
|
498
|
+
* Remember a body until it settles. Never rejects: a failed body is still a finished one, and this
|
|
499
|
+
* set exists to answer "is anything still arriving", not "did it go well".
|
|
500
|
+
*/
|
|
501
|
+
function trackBody(client, completed) {
|
|
502
|
+
const settled = completed.then(
|
|
503
|
+
() => {},
|
|
504
|
+
() => {},
|
|
505
|
+
);
|
|
506
|
+
client._inflight.add(settled);
|
|
507
|
+
settled.then(() => client._inflight.delete(settled));
|
|
508
|
+
}
|
|
509
|
+
|
|
425
510
|
function serverNeverSawIt(err) {
|
|
426
511
|
if (err instanceof UnexpectedEofError) return err.detail?.got === 0;
|
|
427
512
|
return err instanceof TunnelFetchError && err.code === codes.TLS_TRUNCATED && err.detail?.got === 0;
|
|
@@ -493,6 +578,7 @@ async function sendAndReceive(client, conn, current, { key, deadlines, reused })
|
|
|
493
578
|
|
|
494
579
|
// The connection goes back to the pool only when the body reaches the end its framing declared.
|
|
495
580
|
// `completed` resolving false means the caller cancelled and the stream position is unknown.
|
|
581
|
+
trackBody(client, raw.completed);
|
|
496
582
|
raw.completed.then(
|
|
497
583
|
(ok) => {
|
|
498
584
|
deadlines.dispose();
|
|
@@ -556,6 +642,7 @@ async function sendAndReceiveH2(client, h2, current, { deadlines }) {
|
|
|
556
642
|
const raw = head.body;
|
|
557
643
|
// Dispose the per-request deadline once the body is done, however it ends. Unlike h1 there is no
|
|
558
644
|
// pool.release: the connection stays shared and alive for the next stream.
|
|
645
|
+
trackBody(client, raw.completed);
|
|
559
646
|
raw.completed.then(
|
|
560
647
|
() => deadlines.dispose(),
|
|
561
648
|
() => deadlines.dispose(),
|
|
@@ -585,7 +672,7 @@ function buildH2Request(client, current, target) {
|
|
|
585
672
|
|
|
586
673
|
if (!headers.has('accept')) headers.set('accept', '*/*');
|
|
587
674
|
if (!headers.has('accept-encoding') && o.decompress !== false) {
|
|
588
|
-
headers.set('accept-encoding',
|
|
675
|
+
headers.set('accept-encoding', acceptEncodingFor(o.decoders));
|
|
589
676
|
}
|
|
590
677
|
if (client.jar) {
|
|
591
678
|
const cookie = client.jar.headerFor(current.url);
|
|
@@ -624,7 +711,7 @@ function decodeResponseBody(body, headers, options) {
|
|
|
624
711
|
if (options.decompress === false) return body;
|
|
625
712
|
const encoding = headers.get('content-encoding');
|
|
626
713
|
if (!encoding) return body;
|
|
627
|
-
return decodeBody(body, encoding);
|
|
714
|
+
return decodeBody(body, encoding, options.decoders ?? null);
|
|
628
715
|
}
|
|
629
716
|
|
|
630
717
|
function buildResponse(headInfo, body, framing, conn) {
|
|
@@ -669,7 +756,7 @@ function buildHeaders(client, current, target) {
|
|
|
669
756
|
if (!headers.has('accept-encoding') && client.options.decompress !== false) {
|
|
670
757
|
// Never advertise br or zstd: the runtime has no DecompressionStream for either, so the
|
|
671
758
|
// reward for asking would be a body we cannot decode.
|
|
672
|
-
headers.set('accept-encoding',
|
|
759
|
+
headers.set('accept-encoding', acceptEncodingFor(client.options.decoders));
|
|
673
760
|
}
|
|
674
761
|
if (client.jar) {
|
|
675
762
|
const cookie = client.jar.headerFor(current.url);
|
package/src/http2/connection.js
CHANGED
|
@@ -123,7 +123,7 @@ export class Http2Retryable extends Http2Error {}
|
|
|
123
123
|
|
|
124
124
|
export class Http2Connection {
|
|
125
125
|
/**
|
|
126
|
-
* @param {import('
|
|
126
|
+
* @param {import('../tls/connect.js').ByteDuplex | { readable: ReadableStream<Uint8Array>,
|
|
127
127
|
* writable: WritableStream<Uint8Array> }} duplex plaintext transport (a TLS session's
|
|
128
128
|
* plaintextDuplex, or a raw socket for cleartext h2 in tests)
|
|
129
129
|
* @param {Http2ConnectionOptions} [opts]
|
package/src/pool.js
CHANGED
|
@@ -52,15 +52,31 @@ export function poolKey({ scheme, hostname, port, proxy, trust, tls }) {
|
|
|
52
52
|
|
|
53
53
|
function trustFingerprint(trust) {
|
|
54
54
|
const mode = trust?.mode ?? 'system';
|
|
55
|
-
|
|
56
|
-
|
|
55
|
+
// The fingerprint must cover exactly the fields the VERIFIER reads for this mode — no fewer, or
|
|
56
|
+
// a connection validated under one policy serves a request under another; no more, or the pool
|
|
57
|
+
// is split along axes that make no difference. src/trust/index.js states the set per mode by
|
|
58
|
+
// rejecting every other key outright (`forbidKeys`), so this mirrors those lists.
|
|
59
|
+
const rev = `rev=${trust?.revocation ?? 'staple'}`;
|
|
60
|
+
const pins = `pins=${[...(trust?.pins ?? [])].sort().join(',')}`;
|
|
61
|
+
// Anchors matter even when they are absent: omitting them means the system store, and a policy
|
|
62
|
+
// pinned against custom anchors is not the policy pinned against the system store.
|
|
63
|
+
const anchors =
|
|
64
|
+
trust?.anchors === undefined
|
|
65
|
+
? 'anchors=system'
|
|
66
|
+
: `anchors=${trust.anchors.length}:${anchorDigest(trust.anchors)}`;
|
|
67
|
+
|
|
68
|
+
// 'none' forbids anchors/verify/revocation, so pins are the only thing that can vary — and they
|
|
69
|
+
// DO vary: pin-only trust still enforces them.
|
|
70
|
+
if (mode === 'none') return `none|${pins}`;
|
|
57
71
|
if (mode === 'custom') {
|
|
58
72
|
// Two different callbacks are two different policies and we cannot compare functions, so a
|
|
59
73
|
// custom policy never shares a connection. Correct, and cheap: custom trust is rare.
|
|
74
|
+
// ('custom' forbids anchors/pins/revocation — the callback owns the whole decision.)
|
|
60
75
|
return `custom:${customCounter(trust.verify)}`;
|
|
61
76
|
}
|
|
62
|
-
if (mode === '
|
|
63
|
-
if (mode === 'anchors') return `anchors:${(trust.anchors ?? []).length}:${anchorDigest(trust.anchors)}`;
|
|
77
|
+
if (mode === 'system') return `system|${rev}`;
|
|
78
|
+
if (mode === 'anchors') return `anchors:${(trust.anchors ?? []).length}:${anchorDigest(trust.anchors)}|${rev}`;
|
|
79
|
+
if (mode === 'pinned') return `pinned|${pins}|${anchors}|${rev}`;
|
|
64
80
|
return `unknown:${mode}`;
|
|
65
81
|
}
|
|
66
82
|
|
package/src/tls/record.js
CHANGED
|
@@ -81,8 +81,19 @@ async function withGrace(promise, ms) {
|
|
|
81
81
|
/**
|
|
82
82
|
* Keys for one direction: a TLS 1.3 traffic `secret` (key and IV derived per RFC 8446 s7.3,
|
|
83
83
|
* KeyUpdate rotation possible) or raw TLS 1.2 key_block slices (no forward rotation).
|
|
84
|
-
*
|
|
85
|
-
*
|
|
84
|
+
* Exactly one of the two shapes is valid — a 1.3 `secret`, or a 1.2 `key`+`iv` — and _makeState
|
|
85
|
+
* enforces that at runtime. It is spelled with optional members rather than as a union because the
|
|
86
|
+
* setters destructure all four names, and TypeScript cannot destructure a union member that some
|
|
87
|
+
* branch lacks; a union here emits declarations that do not type-check for a consumer.
|
|
88
|
+
* @typedef {{ cipher: number, secret?: Uint8Array, key?: Uint8Array, iv?: Uint8Array }} DirectionKeys
|
|
89
|
+
*/
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* One direction's live protection state. Module-level, not inside the constructor: a typedef
|
|
93
|
+
* declared in a function body is not in scope where the emitted declaration needs it.
|
|
94
|
+
* @typedef {null | { aead: import('./aead.js').Aead, seq: bigint, cipher: number,
|
|
95
|
+
* hash: import('./keyschedule.js').ScheduleHash,
|
|
96
|
+
* secret: Uint8Array | null }} DirectionState
|
|
86
97
|
*/
|
|
87
98
|
|
|
88
99
|
/**
|
|
@@ -113,9 +124,6 @@ export class RecordLayer {
|
|
|
113
124
|
this._onPostHandshake = opts.onPostHandshake ?? null;
|
|
114
125
|
|
|
115
126
|
this._version = TLS13; // semantics selector; setVersion() pins it when negotiated
|
|
116
|
-
/** @typedef {null | { aead: import('./aead.js').Aead, seq: bigint, cipher: number,
|
|
117
|
-
* hash: import('./keyschedule.js').ScheduleHash,
|
|
118
|
-
* secret: Uint8Array | null }} DirectionState */
|
|
119
127
|
/** @type {DirectionState} */
|
|
120
128
|
this._send = null;
|
|
121
129
|
/** @type {DirectionState} */
|
package/types/client/decode.d.ts
CHANGED
|
@@ -1,13 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The Accept-Encoding to send given the caller's extra decoders.
|
|
3
|
+
*
|
|
4
|
+
* Registering a decoder is what makes advertising its coding honest — the two must move together
|
|
5
|
+
* or the client asks for bytes it cannot read. Order is registration order after the built-ins,
|
|
6
|
+
* so a caller matching a browser can produce exactly `gzip, deflate, br, zstd`.
|
|
7
|
+
*
|
|
8
|
+
* @param {Record<string, BodyDecoder> | null | undefined} decoders
|
|
9
|
+
* @returns {string}
|
|
10
|
+
*/
|
|
11
|
+
export function acceptEncodingFor(decoders: Record<string, BodyDecoder> | null | undefined): string;
|
|
1
12
|
/**
|
|
2
13
|
* Undo the response's Content-Encoding.
|
|
3
14
|
*
|
|
15
|
+
/**
|
|
16
|
+
* A caller-supplied content decoder: raw coded bytes in, decoded bytes out. Streaming, so a body
|
|
17
|
+
* is never fully buffered on this client's behalf.
|
|
18
|
+
* @typedef {(stream: ReadableStream<Uint8Array>) => ReadableStream<Uint8Array>} BodyDecoder
|
|
19
|
+
*/
|
|
20
|
+
/**
|
|
4
21
|
* @param {ReadableStream<Uint8Array>} stream the raw body
|
|
5
22
|
* @param {string|null|undefined} contentEncoding the Content-Encoding header value; a
|
|
6
23
|
* comma-separated list names codings in the order the SERVER applied them, so decoding
|
|
7
24
|
* applies them in reverse.
|
|
25
|
+
* @param {Record<string, BodyDecoder> | null} [decoders] caller-supplied codings
|
|
8
26
|
* @returns {ReadableStream<Uint8Array>} decoded bytes
|
|
9
27
|
*/
|
|
10
|
-
export function decodeBody(stream: ReadableStream<Uint8Array>, contentEncoding: string | null | undefined): ReadableStream<Uint8Array>;
|
|
28
|
+
export function decodeBody(stream: ReadableStream<Uint8Array>, contentEncoding: string | null | undefined, decoders?: Record<string, BodyDecoder> | null): ReadableStream<Uint8Array>;
|
|
11
29
|
/**
|
|
12
30
|
* Extract the charset parameter from a Content-Type value, handling quoting and other
|
|
13
31
|
* parameters: `text/html; boundary=x; charset="ISO-8859-4"` -> 'iso-8859-4'.
|
|
@@ -46,9 +64,21 @@ export function charsetFor(contentType: string | null | undefined, bodyPrefix?:
|
|
|
46
64
|
*/
|
|
47
65
|
export function decodeText(bytes: Uint8Array, charset?: string): string;
|
|
48
66
|
/**
|
|
49
|
-
*
|
|
67
|
+
* What the request layer advertises with no extra decoders registered. The target runtime's
|
|
50
68
|
* DecompressionStream supports ONLY gzip / deflate / deflate-raw (verified empirically);
|
|
51
69
|
* advertising `br` or `zstd` would invite bytes we can never decode, turning every response
|
|
52
70
|
* from a brotli-preferring CDN into garbage. Keep this list and decodeBody in lockstep.
|
|
71
|
+
*
|
|
72
|
+
* It is also exactly what curl sends, which matters because this client presents curl's TLS and
|
|
73
|
+
* HTTP/2 fingerprints by default: a browser-shaped handshake paired with a curl-shaped
|
|
74
|
+
* Accept-Encoding is a mismatch a bot detector can read straight off the wire.
|
|
53
75
|
*/
|
|
54
76
|
export const ACCEPT_ENCODING: "gzip, deflate";
|
|
77
|
+
/**
|
|
78
|
+
* Undo the response's Content-Encoding.
|
|
79
|
+
*
|
|
80
|
+
* /**
|
|
81
|
+
* A caller-supplied content decoder: raw coded bytes in, decoded bytes out. Streaming, so a body
|
|
82
|
+
* is never fully buffered on this client's behalf.
|
|
83
|
+
*/
|
|
84
|
+
export type BodyDecoder = (stream: ReadableStream<Uint8Array>) => ReadableStream<Uint8Array>;
|
package/types/client.d.ts
CHANGED
|
@@ -54,8 +54,16 @@ export function install(options?: ClientOptions): () => void;
|
|
|
54
54
|
* one across Clients or to persist it.
|
|
55
55
|
* @property {number} [maxRedirects] default 20.
|
|
56
56
|
* @property {number} [maxBodyBytes] enforced from Content-Length before a byte is read.
|
|
57
|
-
* @property {boolean} [decompress] gzip/deflate. Default true.
|
|
58
|
-
*
|
|
57
|
+
* @property {boolean} [decompress] gzip/deflate. Default true.
|
|
58
|
+
* @property {Record<string, import('./client/decode.js').BodyDecoder>} [decoders] extra
|
|
59
|
+
* content-codings this client can read, e.g. `{ br: (s) => ... }`. Registering one is what
|
|
60
|
+
* makes advertising it honest, so each name is appended to Accept-Encoding — a client that
|
|
61
|
+
* asked for a coding it cannot decode would turn every such response into garbage. `br` and
|
|
62
|
+
* `zstd` are not built in because the runtime's DecompressionStream has neither and this
|
|
63
|
+
* package takes no dependencies; supply your own and the cost, and the supply chain, are
|
|
64
|
+
* yours and visible. Measured on the edge: WASM brotli decodes at about 2x native gzip, and
|
|
65
|
+
* the wire bytes it saves do not pay that back — see the README. The reason to turn it on is
|
|
66
|
+
* matching a browser's Accept-Encoding, not saving CPU.
|
|
59
67
|
* @property {boolean} [keepAlive] default true.
|
|
60
68
|
* @property {boolean} [http2] offer HTTP/2 via ALPN and speak it when the server selects it.
|
|
61
69
|
* Default true. The goal is ACCESS, not speed — some sites treat HTTP/1.1 as a bot signal — and
|
|
@@ -75,95 +83,7 @@ export class Client {
|
|
|
75
83
|
* @param {ClientOptions} [options]
|
|
76
84
|
*/
|
|
77
85
|
constructor(options?: ClientOptions);
|
|
78
|
-
options:
|
|
79
|
-
/**
|
|
80
|
-
* Socket factory. Required for any
|
|
81
|
-
* request the platform's own `fetch` cannot serve — which is every proxied request, and every
|
|
82
|
-
* request asking for a trust policy `fetch` cannot express.
|
|
83
|
-
*/
|
|
84
|
-
connect?: import("./proxy/index.js").ConnectFn | undefined;
|
|
85
|
-
/**
|
|
86
|
-
* URL string or
|
|
87
|
-
* config object; `http:`, `https:`, `socks5:` and `socks5h:`.
|
|
88
|
-
*/
|
|
89
|
-
proxy?: string | import("./proxy/index.js").ProxyConfig | null | undefined;
|
|
90
|
-
/**
|
|
91
|
-
* certificate policy, httpx's
|
|
92
|
-
* `verify=`. Default `{ mode: 'system' }`.
|
|
93
|
-
*/
|
|
94
|
-
trust?: import("./trust/index.js").TrustConfig | undefined;
|
|
95
|
-
/**
|
|
96
|
-
* handshake knobs.
|
|
97
|
-
*/
|
|
98
|
-
tls?: import("./tls/connect.js").TlsOptions | undefined;
|
|
99
|
-
/**
|
|
100
|
-
* connect / handshake /
|
|
101
|
-
* headers / idle / total, in ms. The idle gap is the control; total is a backstop.
|
|
102
|
-
*/
|
|
103
|
-
timeouts?: import("./util/deadline.js").DeadlineOptions | undefined;
|
|
104
|
-
/**
|
|
105
|
-
* enable a per-Client cookie jar.
|
|
106
|
-
*/
|
|
107
|
-
cookies?: boolean | undefined;
|
|
108
|
-
/**
|
|
109
|
-
* supply a jar directly, e.g. to share
|
|
110
|
-
* one across Clients or to persist it.
|
|
111
|
-
*/
|
|
112
|
-
jar?: CookieJar | undefined;
|
|
113
|
-
/**
|
|
114
|
-
* default 20.
|
|
115
|
-
*/
|
|
116
|
-
maxRedirects?: number | undefined;
|
|
117
|
-
/**
|
|
118
|
-
* enforced from Content-Length before a byte is read.
|
|
119
|
-
*/
|
|
120
|
-
maxBodyBytes?: number | undefined;
|
|
121
|
-
/**
|
|
122
|
-
* gzip/deflate. Default true. Never `br`; the runtime cannot
|
|
123
|
-
* decompress it, so it is never advertised either.
|
|
124
|
-
*/
|
|
125
|
-
decompress?: boolean | undefined;
|
|
126
|
-
/**
|
|
127
|
-
* default true.
|
|
128
|
-
*/
|
|
129
|
-
keepAlive?: boolean | undefined;
|
|
130
|
-
/**
|
|
131
|
-
* offer HTTP/2 via ALPN and speak it when the server selects it.
|
|
132
|
-
* Default true. The goal is ACCESS, not speed — some sites treat HTTP/1.1 as a bot signal — and
|
|
133
|
-
* on a CPU-billed runtime h2 costs MORE than h1 (HPACK is extra work). Set false to offer only
|
|
134
|
-
* `http/1.1`. There is no fallback-and-retry either way: the server's ALPN pick is followed.
|
|
135
|
-
*/
|
|
136
|
-
http2?: boolean | undefined;
|
|
137
|
-
/**
|
|
138
|
-
* never delegate to the platform's fetch, even when it could
|
|
139
|
-
* serve the request. Mainly for exercising this stack against origins that do not need it.
|
|
140
|
-
*/
|
|
141
|
-
forceTunnel?: boolean | undefined;
|
|
142
|
-
/**
|
|
143
|
-
* delegation target; defaults to `globalThis.fetch`.
|
|
144
|
-
*/
|
|
145
|
-
nativeFetch?: FetchLike | undefined;
|
|
146
|
-
/**
|
|
147
|
-
* connection pool sizing.
|
|
148
|
-
*/
|
|
149
|
-
pool?: {
|
|
150
|
-
maxPerKey?: number;
|
|
151
|
-
maxTotal?: number;
|
|
152
|
-
} | undefined;
|
|
153
|
-
limits?: Limits | undefined;
|
|
154
|
-
/**
|
|
155
|
-
* injectable randomness and key generation.
|
|
156
|
-
*/
|
|
157
|
-
deps?: import("./tls/connect.js").TlsDeps | undefined;
|
|
158
|
-
/**
|
|
159
|
-
* aborts every request this Client makes.
|
|
160
|
-
*/
|
|
161
|
-
signal?: AbortSignal | undefined;
|
|
162
|
-
/**
|
|
163
|
-
* epoch ms override, for certificate validity in tests.
|
|
164
|
-
*/
|
|
165
|
-
now?: number | undefined;
|
|
166
|
-
};
|
|
86
|
+
options: Readonly<ClientOptions>;
|
|
167
87
|
pool: ConnectionPool;
|
|
168
88
|
/** @type {Map<string, import('./http2/connection.js').Http2Connection>} */
|
|
169
89
|
_h2: Map<string, import("./http2/connection.js").Http2Connection>;
|
|
@@ -172,6 +92,8 @@ export class Client {
|
|
|
172
92
|
jar: CookieJar | null;
|
|
173
93
|
tickets: TicketStore;
|
|
174
94
|
_closed: boolean;
|
|
95
|
+
/** @type {Set<Promise<void>>} */
|
|
96
|
+
_inflight: Set<Promise<void>>;
|
|
175
97
|
/**
|
|
176
98
|
* @param {RequestInfo | URL} input
|
|
177
99
|
* @param {RequestInit} [init]
|
|
@@ -180,6 +102,16 @@ export class Client {
|
|
|
180
102
|
fetch(input: RequestInfo | URL, init?: RequestInit): Promise<Response & {
|
|
181
103
|
tunnelfetch?: ResponseDetail;
|
|
182
104
|
}>;
|
|
105
|
+
/**
|
|
106
|
+
* Resolve once every response body handed out by this Client has finished, one way or another —
|
|
107
|
+
* read to the end, cancelled, or failed. Deliberately NOT folded into close(): close() is the
|
|
108
|
+
* forceful teardown, and a teardown that waits on the streams it is tearing down would hang on
|
|
109
|
+
* any body the caller abandoned. This is for callers that want the graceful order.
|
|
110
|
+
*
|
|
111
|
+
* Looping rather than a single Promise.all because a body settling can start another (a redirect
|
|
112
|
+
* drains its predecessor), and a set sampled once would miss the successor.
|
|
113
|
+
*/
|
|
114
|
+
idle(): Promise<void>;
|
|
183
115
|
/** Release every pooled socket and shared HTTP/2 connection. A Client that is finished must be
|
|
184
116
|
* closed or sockets leak for the isolate's lifetime. */
|
|
185
117
|
close(): Promise<void>;
|
|
@@ -271,10 +203,21 @@ export type ClientOptions = {
|
|
|
271
203
|
*/
|
|
272
204
|
maxBodyBytes?: number | undefined;
|
|
273
205
|
/**
|
|
274
|
-
* gzip/deflate. Default true.
|
|
275
|
-
* decompress it, so it is never advertised either.
|
|
206
|
+
* gzip/deflate. Default true.
|
|
276
207
|
*/
|
|
277
208
|
decompress?: boolean | undefined;
|
|
209
|
+
/**
|
|
210
|
+
* extra
|
|
211
|
+
* content-codings this client can read, e.g. `{ br: (s) => ... }`. Registering one is what
|
|
212
|
+
* makes advertising it honest, so each name is appended to Accept-Encoding — a client that
|
|
213
|
+
* asked for a coding it cannot decode would turn every such response into garbage. `br` and
|
|
214
|
+
* `zstd` are not built in because the runtime's DecompressionStream has neither and this
|
|
215
|
+
* package takes no dependencies; supply your own and the cost, and the supply chain, are
|
|
216
|
+
* yours and visible. Measured on the edge: WASM brotli decodes at about 2x native gzip, and
|
|
217
|
+
* the wire bytes it saves do not pay that back — see the README. The reason to turn it on is
|
|
218
|
+
* matching a browser's Accept-Encoding, not saving CPU.
|
|
219
|
+
*/
|
|
220
|
+
decoders?: Record<string, import("./client/decode.js").BodyDecoder> | undefined;
|
|
278
221
|
/**
|
|
279
222
|
* default true.
|
|
280
223
|
*/
|
|
@@ -316,8 +259,8 @@ export type ClientOptions = {
|
|
|
316
259
|
*/
|
|
317
260
|
now?: number | undefined;
|
|
318
261
|
};
|
|
319
|
-
import { CookieJar } from './client/cookies.js';
|
|
320
262
|
import { ConnectionPool } from './pool.js';
|
|
263
|
+
import { CookieJar } from './client/cookies.js';
|
|
321
264
|
import { TicketStore } from './tls/tickets.js';
|
|
322
265
|
import { utf8 } from './util/bytes.js';
|
|
323
266
|
export { CookieJar, ConnectionPool, utf8 };
|
|
@@ -49,12 +49,12 @@ export class Http2Retryable extends Http2Error {
|
|
|
49
49
|
*/
|
|
50
50
|
export class Http2Connection {
|
|
51
51
|
/**
|
|
52
|
-
* @param {import('
|
|
52
|
+
* @param {import('../tls/connect.js').ByteDuplex | { readable: ReadableStream<Uint8Array>,
|
|
53
53
|
* writable: WritableStream<Uint8Array> }} duplex plaintext transport (a TLS session's
|
|
54
54
|
* plaintextDuplex, or a raw socket for cleartext h2 in tests)
|
|
55
55
|
* @param {Http2ConnectionOptions} [opts]
|
|
56
56
|
*/
|
|
57
|
-
constructor(duplex: import("
|
|
57
|
+
constructor(duplex: import("../tls/connect.js").ByteDuplex | {
|
|
58
58
|
readable: ReadableStream<Uint8Array>;
|
|
59
59
|
writable: WritableStream<Uint8Array>;
|
|
60
60
|
}, opts?: Http2ConnectionOptions);
|
package/types/tls/record.d.ts
CHANGED
|
@@ -9,8 +9,18 @@
|
|
|
9
9
|
/**
|
|
10
10
|
* Keys for one direction: a TLS 1.3 traffic `secret` (key and IV derived per RFC 8446 s7.3,
|
|
11
11
|
* KeyUpdate rotation possible) or raw TLS 1.2 key_block slices (no forward rotation).
|
|
12
|
-
*
|
|
13
|
-
*
|
|
12
|
+
* Exactly one of the two shapes is valid — a 1.3 `secret`, or a 1.2 `key`+`iv` — and _makeState
|
|
13
|
+
* enforces that at runtime. It is spelled with optional members rather than as a union because the
|
|
14
|
+
* setters destructure all four names, and TypeScript cannot destructure a union member that some
|
|
15
|
+
* branch lacks; a union here emits declarations that do not type-check for a consumer.
|
|
16
|
+
* @typedef {{ cipher: number, secret?: Uint8Array, key?: Uint8Array, iv?: Uint8Array }} DirectionKeys
|
|
17
|
+
*/
|
|
18
|
+
/**
|
|
19
|
+
* One direction's live protection state. Module-level, not inside the constructor: a typedef
|
|
20
|
+
* declared in a function body is not in scope where the emitted declaration needs it.
|
|
21
|
+
* @typedef {null | { aead: import('./aead.js').Aead, seq: bigint, cipher: number,
|
|
22
|
+
* hash: import('./keyschedule.js').ScheduleHash,
|
|
23
|
+
* secret: Uint8Array | null }} DirectionState
|
|
14
24
|
*/
|
|
15
25
|
/**
|
|
16
26
|
* @typedef {object} RecordLayerOptions
|
|
@@ -37,25 +47,10 @@ export class RecordLayer {
|
|
|
37
47
|
_padding: ((type: number, length: number) => number) | null;
|
|
38
48
|
_onPostHandshake: ((msg: HandshakeMessage) => void | Promise<void>) | null;
|
|
39
49
|
_version: number;
|
|
40
|
-
/** @typedef {null | { aead: import('./aead.js').Aead, seq: bigint, cipher: number,
|
|
41
|
-
* hash: import('./keyschedule.js').ScheduleHash,
|
|
42
|
-
* secret: Uint8Array | null }} DirectionState */
|
|
43
50
|
/** @type {DirectionState} */
|
|
44
|
-
_send:
|
|
45
|
-
aead: import("./aead.js").Aead;
|
|
46
|
-
seq: bigint;
|
|
47
|
-
cipher: number;
|
|
48
|
-
hash: import("./keyschedule.js").ScheduleHash;
|
|
49
|
-
secret: Uint8Array | null;
|
|
50
|
-
} | null;
|
|
51
|
+
_send: DirectionState;
|
|
51
52
|
/** @type {DirectionState} */
|
|
52
|
-
_recv:
|
|
53
|
-
aead: import("./aead.js").Aead;
|
|
54
|
-
seq: bigint;
|
|
55
|
-
cipher: number;
|
|
56
|
-
hash: import("./keyschedule.js").ScheduleHash;
|
|
57
|
-
secret: Uint8Array | null;
|
|
58
|
-
} | null;
|
|
53
|
+
_recv: DirectionState;
|
|
59
54
|
/** Handshake reassembly. Chunk list, not one growing buffer, so a peer drip-feeding a
|
|
60
55
|
* message in tiny records costs O(records), not O(records^2).
|
|
61
56
|
* @type {Uint8Array[]} */
|
|
@@ -323,14 +318,27 @@ export type HandshakeMessage = {
|
|
|
323
318
|
/**
|
|
324
319
|
* Keys for one direction: a TLS 1.3 traffic `secret` (key and IV derived per RFC 8446 s7.3,
|
|
325
320
|
* KeyUpdate rotation possible) or raw TLS 1.2 key_block slices (no forward rotation).
|
|
321
|
+
* Exactly one of the two shapes is valid — a 1.3 `secret`, or a 1.2 `key`+`iv` — and _makeState
|
|
322
|
+
* enforces that at runtime. It is spelled with optional members rather than as a union because the
|
|
323
|
+
* setters destructure all four names, and TypeScript cannot destructure a union member that some
|
|
324
|
+
* branch lacks; a union here emits declarations that do not type-check for a consumer.
|
|
326
325
|
*/
|
|
327
326
|
export type DirectionKeys = {
|
|
328
327
|
cipher: number;
|
|
329
|
-
secret
|
|
330
|
-
|
|
328
|
+
secret?: Uint8Array;
|
|
329
|
+
key?: Uint8Array;
|
|
330
|
+
iv?: Uint8Array;
|
|
331
|
+
};
|
|
332
|
+
/**
|
|
333
|
+
* One direction's live protection state. Module-level, not inside the constructor: a typedef
|
|
334
|
+
* declared in a function body is not in scope where the emitted declaration needs it.
|
|
335
|
+
*/
|
|
336
|
+
export type DirectionState = null | {
|
|
337
|
+
aead: import("./aead.js").Aead;
|
|
338
|
+
seq: bigint;
|
|
331
339
|
cipher: number;
|
|
332
|
-
|
|
333
|
-
|
|
340
|
+
hash: import("./keyschedule.js").ScheduleHash;
|
|
341
|
+
secret: Uint8Array | null;
|
|
334
342
|
};
|
|
335
343
|
export type RecordLayerOptions = {
|
|
336
344
|
/**
|