tunnelfetch 1.1.2 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +44 -1
- package/README.zh-CN.md +32 -0
- package/package.json +1 -1
- package/src/client/header-order.js +159 -0
- package/src/client.js +62 -16
- package/src/http2/connection.js +55 -9
- package/src/index.js +1 -0
- package/src/profiles.js +150 -0
- package/src/tls/connect.js +14 -0
- package/src/tls/extensions.js +10 -0
- package/src/tls/grease.js +109 -0
- package/src/tls/handshake-messages.js +120 -4
- package/src/tls/handshake.js +5 -0
- package/src/warmup-fixture.js +44 -44
- package/types/client/header-order.d.ts +56 -0
- package/types/client.d.ts +53 -0
- package/types/http2/connection.d.ts +25 -1
- package/types/http2/hpack.d.ts +1 -1
- package/types/index.d.ts +1 -0
- package/types/profiles.d.ts +96 -0
- package/types/tls/connect.d.ts +31 -0
- package/types/tls/extensions.d.ts +7 -0
- package/types/tls/grease.d.ts +46 -0
- package/types/tls/handshake-messages.d.ts +104 -57
package/README.md
CHANGED
|
@@ -311,10 +311,53 @@ behaviour — measure your own targets, and expect the answer to change. Our TLS
|
|
|
311
311
|
identical outcomes on every reachable host in that sample, so JA3-style TLS shaping is not what
|
|
312
312
|
gates access here — but curl's **HTTP/2** fingerprint passed where HTTP/1.1 was challenged. So the
|
|
313
313
|
`SETTINGS` frame values, the initial window sizes, the connection `WINDOW_UPDATE`, and the
|
|
314
|
-
pseudo-header order are matched byte-for-byte to curl (
|
|
314
|
+
pseudo-header order are matched byte-for-byte to curl (nghttp2 1.69.0), captured off the wire.
|
|
315
315
|
This is empirical: a naïve h2 fingerprint can fail exactly where curl's succeeds, which would waste
|
|
316
316
|
the whole exercise.
|
|
317
317
|
|
|
318
|
+
### Fingerprints
|
|
319
|
+
|
|
320
|
+
Both halves are matched to **curl 8.21.0 / OpenSSL 3.6.3** and both are fully configurable. The TLS
|
|
321
|
+
backend matters more than the curl version — the same curl built against SecureTransport produces a
|
|
322
|
+
completely different ClientHello — so the reference is captured off the wire, not recalled, and
|
|
323
|
+
pinned in `test/tls/fingerprint.test.js` and `test/http2/fingerprint.test.js`.
|
|
324
|
+
|
|
325
|
+
| Layer | Default | Configure with |
|
|
326
|
+
|---|---|---|
|
|
327
|
+
| ClientHello extension **order** | curl's, exactly, for every extension both send | `tls.extensionOrder` |
|
|
328
|
+
| Cipher suites | the AEAD suites this package implements, in curl's relative order | `tls.ciphers` |
|
|
329
|
+
| Supported groups | `x25519, secp256r1, secp384r1, secp521r1` | `tls.groups` |
|
|
330
|
+
| Signature algorithms | ECDSA and RSA-PSS/PKCS#1 over SHA-256/384/512 | `tls.sigSchemes` |
|
|
331
|
+
| ALPN | `h2, http/1.1` | `tls.alpn` |
|
|
332
|
+
| HTTP/2 `SETTINGS` ids **and order** | curl's: `MAX_CONCURRENT_STREAMS, INITIAL_WINDOW_SIZE, ENABLE_PUSH` | `http2Settings` |
|
|
333
|
+
| h2 preface, `WINDOW_UPDATE`, pseudo-header order, HPACK representation | curl's, byte-for-byte | fixed |
|
|
334
|
+
| `Accept-Encoding` | `gzip, deflate` — curl's | `decoders` appends |
|
|
335
|
+
|
|
336
|
+
Extension order matters because JA3 and JA4 hash the extension list **in wire order**, so it is most
|
|
337
|
+
of what a fingerprinter reads. `pre_shared_key` is forced last whatever you ask for: RFC 8446
|
|
338
|
+
§4.2.11 defines the binder transcript as the hello truncated just before the binders, which is a
|
|
339
|
+
well-defined byte range only if nothing follows them.
|
|
340
|
+
|
|
341
|
+
**Where the default deliberately differs from curl**, and why it cannot simply be copied: a
|
|
342
|
+
ClientHello is an *offer*, and a server may take you up on any of it. Advertising what you cannot do
|
|
343
|
+
trades a fingerprint mismatch for a broken handshake, which is worse and fails silently.
|
|
344
|
+
|
|
345
|
+
| curl sends | This package | Why |
|
|
346
|
+
|---|---|---|
|
|
347
|
+
| 30 cipher suites, incl. RSA key exchange and CBC | 6 AEAD suites | Not implemented, by design. A server selecting `TLS_RSA_WITH_AES_256_CBC_SHA` would get a dead connection |
|
|
348
|
+
| `X25519MLKEM768` group and a 1216-byte key share | not offered | ML-KEM is not implemented |
|
|
349
|
+
| SHA-1 signature schemes | not offered | Refused deliberately |
|
|
350
|
+
| `encrypt_then_mac` | not sent | Applies only to CBC suites, which are not offered |
|
|
351
|
+
| `post_handshake_auth` | not sent | Invites a post-handshake `CertificateRequest`, which is not implemented |
|
|
352
|
+
| — | `status_request` | curl does not ask for a stapled OCSP response; this package must, because a staple is its only revocation signal |
|
|
353
|
+
|
|
354
|
+
Closing that gap means implementing ML-KEM, RSA key exchange and CBC suites — a different project,
|
|
355
|
+
and the last two are things this package refuses on purpose. `tls.ciphers` and `tls.groups` will let
|
|
356
|
+
you offer them anyway; the handshake will then fail if a server picks one, and that is yours to own.
|
|
357
|
+
|
|
358
|
+
A test asserts this delta is exactly the list above, so gaining one of these capabilities without
|
|
359
|
+
updating the table fails the build.
|
|
360
|
+
|
|
318
361
|
Everything an HTTP/1.1 body has, an HTTP/2 body keeps: streaming (SSE works unchanged), trailers,
|
|
319
362
|
gzip decoding, and the idle deadline wrapping the raw body before any decode. The one thing that is
|
|
320
363
|
structurally different is under the hood — a single h2 connection multiplexes every concurrent
|
package/README.zh-CN.md
CHANGED
|
@@ -251,6 +251,38 @@ TLS 指纹塑形不是这里的门槛——但 curl 的 **HTTP/2** 指纹能过
|
|
|
251
251
|
curl(8.7.1 / nghttp2),照线上抓包原样复刻。这么做是实证需要,不是讲究:一个想当然的 h2
|
|
252
252
|
指纹,可能恰好败在 curl 能过的地方——那这整件事就白做了。
|
|
253
253
|
|
|
254
|
+
### 指纹
|
|
255
|
+
|
|
256
|
+
两侧都对齐 **curl 8.21.0 / OpenSSL 3.6.3**,且全部可自定义。TLS 后端比 curl 版本更要紧——同一个 curl 用 SecureTransport 编译出来的 ClientHello 完全不同——所以参照物是从线上抓下来的,不是凭记忆写的,并且钉死在 `test/tls/fingerprint.test.js` 与 `test/http2/fingerprint.test.js` 里。
|
|
257
|
+
|
|
258
|
+
| 层 | 默认 | 配置项 |
|
|
259
|
+
|---|---|---|
|
|
260
|
+
| ClientHello 扩展**顺序** | 与 curl 完全一致(就双方都发的那些而言) | `tls.extensionOrder` |
|
|
261
|
+
| 密码套件 | 本包实现的 AEAD 套件,按 curl 的相对顺序 | `tls.ciphers` |
|
|
262
|
+
| supported_groups | `x25519, secp256r1, secp384r1, secp521r1` | `tls.groups` |
|
|
263
|
+
| 签名算法 | SHA-256/384/512 上的 ECDSA 与 RSA-PSS/PKCS#1 | `tls.sigSchemes` |
|
|
264
|
+
| ALPN | `h2, http/1.1` | `tls.alpn` |
|
|
265
|
+
| HTTP/2 `SETTINGS` 的 id **与顺序** | curl 的:`MAX_CONCURRENT_STREAMS, INITIAL_WINDOW_SIZE, ENABLE_PUSH` | `http2Settings` |
|
|
266
|
+
| h2 前导、`WINDOW_UPDATE`、伪头顺序、HPACK 表示 | curl 的,逐字节一致 | 固定 |
|
|
267
|
+
| `Accept-Encoding` | `gzip, deflate`——curl 的 | `decoders` 会追加 |
|
|
268
|
+
|
|
269
|
+
扩展顺序之所以要紧,是因为 JA3 和 JA4 哈希的正是**线上顺序**的扩展列表,那是指纹识别读到的主要内容。`pre_shared_key` 无论你怎么配都强制排最后:RFC 8446 §4.2.11 把 binder 的转录定义为"截到 binder 之前的那段 hello",只有后面不跟东西时这个范围才成立。
|
|
270
|
+
|
|
271
|
+
**默认值刻意与 curl 不同的地方**,以及为什么不能照抄:ClientHello 是一份**要约**,服务器可以接受其中任何一项。声明你做不到的事,等于拿指纹不一致换一次握手失败——后者更糟,而且是静默的。
|
|
272
|
+
|
|
273
|
+
| curl 发送 | 本包 | 原因 |
|
|
274
|
+
|---|---|---|
|
|
275
|
+
| 30 个套件,含 RSA 密钥交换与 CBC | 6 个 AEAD 套件 | 有意不实现。服务器选中 `TLS_RSA_WITH_AES_256_CBC_SHA` 会得到一条死连接 |
|
|
276
|
+
| `X25519MLKEM768` 群及 1216 字节 key share | 不提供 | 未实现 ML-KEM |
|
|
277
|
+
| SHA-1 签名方案 | 不提供 | 明确拒绝 |
|
|
278
|
+
| `encrypt_then_mac` | 不发 | 只对 CBC 套件有意义,而 CBC 不提供 |
|
|
279
|
+
| `post_handshake_auth` | 不发 | 会招来握手后的 `CertificateRequest`,未实现 |
|
|
280
|
+
| — | `status_request` | curl 不索要 OCSP staple;本包必须要,因为 staple 是它唯一的吊销信号 |
|
|
281
|
+
|
|
282
|
+
要抹平这个差距,得先实现 ML-KEM、RSA 密钥交换和 CBC 套件——那是另一个项目,而且后两者是本包**故意**不做的。`tls.ciphers` 和 `tls.groups` 允许你照样把它们报出去;服务器一旦选中握手就会失败,后果由你自己承担。
|
|
283
|
+
|
|
284
|
+
有一个测试断言这张差异表**恰好**就是上面这些,所以哪天获得了其中某项能力却没更新这张表,构建会直接失败。
|
|
285
|
+
|
|
254
286
|
HTTP/1.1 的 body 有的,HTTP/2 的 body 全都保留:流式(SSE 原样可用)、trailer、gzip 解码、
|
|
255
287
|
以及在任何解码之前先包住原始 body 的 idle 截止线。结构上唯一不同的在水面之下——一条 h2 连接把
|
|
256
288
|
发往同一源站的所有并发请求复用在一起,而不是一次签出、一次只服务一个请求。`install()`、重定向、
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "tunnelfetch",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.3.0",
|
|
4
4
|
"description": "A fetch-shaped HTTP client that can route through HTTP CONNECT / HTTPS / SOCKS5 proxies on runtimes with only raw TCP, such as Cloudflare Workers. Implements TLS in userland because the runtime cannot verify a tunnelled peer.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"fetch",
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
// Request header order and case — the part of a fingerprint that needs no TLS inspection at all.
|
|
2
|
+
//
|
|
3
|
+
// The platform's `Headers` is the wrong wire representation and always was: it iterates
|
|
4
|
+
// lexicographically sorted and lowercased. So a request built through it goes out as
|
|
5
|
+
// `accept, accept-encoding, connection, host, referer, user-agent` whatever the caller wrote —
|
|
6
|
+
// alphabetical order, with `user-agent` last, which no real client does. curl sends:
|
|
7
|
+
//
|
|
8
|
+
// Host, User-Agent, Accept, Accept-Encoding, <the caller's headers, in order>,
|
|
9
|
+
// Content-Length, Content-Type
|
|
10
|
+
//
|
|
11
|
+
// captured off the wire from curl 8.21.0. Note where the framing headers go: last, AFTER the
|
|
12
|
+
// caller's. That is why the default order below carries a `'*'` marker rather than being a plain
|
|
13
|
+
// ranking — "everything else" belongs in the middle, not at the end.
|
|
14
|
+
//
|
|
15
|
+
// Case matters too, and only on HTTP/1.1: real clients send `Host:` and `X-Custom:`, not `host:`
|
|
16
|
+
// and `x-custom:`. HTTP/2 is the opposite — RFC 9113 s8.2.1 REQUIRES lowercase field names, and a
|
|
17
|
+
// server must treat an uppercase one as malformed. So this preserves the case it was given and the
|
|
18
|
+
// h2 path lowercases at the last moment, which is the only place that is correct.
|
|
19
|
+
|
|
20
|
+
import { HttpError, codes } from '../errors.js';
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Default request header order, from curl 8.21.0 on the wire. `'*'` is where headers not named
|
|
24
|
+
* here go, in the order they were given — which is where curl puts the caller's own.
|
|
25
|
+
*/
|
|
26
|
+
export const CURL_HEADER_ORDER = Object.freeze([
|
|
27
|
+
'host',
|
|
28
|
+
'user-agent',
|
|
29
|
+
'accept',
|
|
30
|
+
'accept-encoding',
|
|
31
|
+
'*',
|
|
32
|
+
// curl sends no Connection on HTTP/1.1 (keep-alive is the default and it stays silent), so
|
|
33
|
+
// there is no reference position for it. Grouped with the other connection-and-framing headers
|
|
34
|
+
// at the end, which is where curl puts the ones it does send.
|
|
35
|
+
'connection',
|
|
36
|
+
'content-length',
|
|
37
|
+
'content-type',
|
|
38
|
+
]);
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* An ordered, case-preserving header list.
|
|
42
|
+
*
|
|
43
|
+
* Deliberately not a `Headers` subclass and deliberately not backed by one: the whole point is to
|
|
44
|
+
* be the thing `Headers` is not. Lookup and mutation are case-insensitive, as HTTP requires;
|
|
45
|
+
* iteration returns names exactly as they were written, in the order they arrived.
|
|
46
|
+
*/
|
|
47
|
+
export class OrderedHeaders {
|
|
48
|
+
/** @param {Headers | Iterable<[string, string]> | Record<string, string> | null} [init] */
|
|
49
|
+
constructor(init = null) {
|
|
50
|
+
/** @type {Array<[string, string]>} name as written, value */
|
|
51
|
+
this._list = [];
|
|
52
|
+
if (init == null) return;
|
|
53
|
+
const pairs =
|
|
54
|
+
typeof (/** @type {any} */ (init)[Symbol.iterator]) === 'function'
|
|
55
|
+
? /** @type {Iterable<[string, string]>} */ (init)
|
|
56
|
+
: Object.entries(init);
|
|
57
|
+
for (const [name, value] of pairs) this.append(name, value);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
_indexOf(name) {
|
|
61
|
+
const lower = String(name).toLowerCase();
|
|
62
|
+
return this._list.findIndex(([n]) => n.toLowerCase() === lower);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
has(name) {
|
|
66
|
+
return this._indexOf(name) !== -1;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** Comma-joined when a field appears more than once, matching `Headers.get`. */
|
|
70
|
+
get(name) {
|
|
71
|
+
const lower = String(name).toLowerCase();
|
|
72
|
+
const hits = this._list.filter(([n]) => n.toLowerCase() === lower).map(([, v]) => v);
|
|
73
|
+
return hits.length ? hits.join(', ') : null;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Replace IN PLACE when the field is already present, so setting a value does not move a header
|
|
78
|
+
* to the end and silently reorder the request. A caller who wrote `User-Agent` first and then
|
|
79
|
+
* had it overwritten should still see it first.
|
|
80
|
+
*/
|
|
81
|
+
set(name, value) {
|
|
82
|
+
const at = this._indexOf(name);
|
|
83
|
+
if (at === -1) {
|
|
84
|
+
this.append(name, value);
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
87
|
+
this._list[at] = [this._list[at][0], String(value)];
|
|
88
|
+
// A repeated field collapses to the first position, as `Headers.set` collapses to one value.
|
|
89
|
+
const lower = String(name).toLowerCase();
|
|
90
|
+
this._list = this._list.filter(([n], i) => i === at || n.toLowerCase() !== lower);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
append(name, value) {
|
|
94
|
+
const n = String(name);
|
|
95
|
+
if (n === '') throw new HttpError(codes.HTTP_HEADER, 'header name must not be empty', { name });
|
|
96
|
+
this._list.push([n, String(value)]);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
delete(name) {
|
|
100
|
+
const lower = String(name).toLowerCase();
|
|
101
|
+
this._list = this._list.filter(([n]) => n.toLowerCase() !== lower);
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Put the fields into `order`, which names lowercased header names and may contain `'*'` to mark
|
|
106
|
+
* where everything unnamed goes. Fields keep their relative order within each group, so a
|
|
107
|
+
* caller's own sequence survives.
|
|
108
|
+
*
|
|
109
|
+
* @param {readonly string[]} order
|
|
110
|
+
*/
|
|
111
|
+
reorder(order) {
|
|
112
|
+
const star = order.indexOf('*');
|
|
113
|
+
const rank = new Map();
|
|
114
|
+
order.forEach((name, i) => {
|
|
115
|
+
if (name !== '*') rank.set(name, i);
|
|
116
|
+
});
|
|
117
|
+
// With no '*', unnamed fields go last — the same rule the TLS extension order uses.
|
|
118
|
+
const fallback = star === -1 ? order.length : star;
|
|
119
|
+
this._list = this._list
|
|
120
|
+
.map((entry, i) => ({ entry, i, r: rank.get(entry[0].toLowerCase()) ?? fallback }))
|
|
121
|
+
.sort((a, b) => a.r - b.r || a.i - b.i)
|
|
122
|
+
.map((x) => x.entry);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/** @returns {Array<[string, string]>} names as written, in order */
|
|
126
|
+
entries() {
|
|
127
|
+
return this._list.map(([n, v]) => [n, v]);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/** Lowercased names, for HTTP/2 where RFC 9113 s8.2.1 requires them. */
|
|
131
|
+
lowercased() {
|
|
132
|
+
return this._list.map(([n, v]) => [n.toLowerCase(), v]);
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
[Symbol.iterator]() {
|
|
136
|
+
return this.entries()[Symbol.iterator]();
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* Read the caller's header names in the order and case they wrote them, BEFORE anything hands them
|
|
142
|
+
* to `Request`, which is where both are lost.
|
|
143
|
+
*
|
|
144
|
+
* Recovers what is recoverable and no more: an array of pairs or a plain object still carries the
|
|
145
|
+
* caller's order, a `Headers` or a `Request` does not — those were normalised before this package
|
|
146
|
+
* ever saw them, and there is nothing here to reconstruct.
|
|
147
|
+
*
|
|
148
|
+
* @param {RequestInfo | URL} input
|
|
149
|
+
* @param {RequestInit} [init]
|
|
150
|
+
* @returns {Array<[string, string]> | null} null when the caller's order was already gone
|
|
151
|
+
*/
|
|
152
|
+
export function callerHeaderOrder(input, init) {
|
|
153
|
+
const raw = init?.headers ?? (input instanceof Request ? null : undefined);
|
|
154
|
+
if (raw == null) return null;
|
|
155
|
+
if (typeof Headers !== 'undefined' && raw instanceof Headers) return null;
|
|
156
|
+
if (Array.isArray(raw)) return raw.map(([n, v]) => [String(n), String(v)]);
|
|
157
|
+
if (typeof raw === 'object') return Object.entries(raw).map(([n, v]) => [n, String(v)]);
|
|
158
|
+
return null;
|
|
159
|
+
}
|
package/src/client.js
CHANGED
|
@@ -15,6 +15,8 @@ import { ByteReader, ByteWriter, UnexpectedEofError, concat, utf8 } from './util
|
|
|
15
15
|
import { serializeRequestHead } from './http1/request.js';
|
|
16
16
|
import { bodyFraming, readResponseBody, readResponseHead } from './http1/response.js';
|
|
17
17
|
import { acceptEncodingFor, decodeBody } from './client/decode.js';
|
|
18
|
+
import { OrderedHeaders, CURL_HEADER_ORDER, callerHeaderOrder } from './client/header-order.js';
|
|
19
|
+
import { applyProfile } from './profiles.js';
|
|
18
20
|
import { CookieJar } from './client/cookies.js';
|
|
19
21
|
import { DEFAULT_MAX_REDIRECTS, nextRequest, shouldRedirect } from './client/redirect.js';
|
|
20
22
|
import { ConnectionPool, poolKey } from './pool.js';
|
|
@@ -82,6 +84,25 @@ const NULL_BODY_STATUS = new Set([101, 204, 205, 304]);
|
|
|
82
84
|
* the wire bytes it saves do not pay that back — see the README. The reason to turn it on is
|
|
83
85
|
* matching a browser's Accept-Encoding, not saving CPU.
|
|
84
86
|
* @property {boolean} [keepAlive] default true.
|
|
87
|
+
* @property {import('./profiles.js').FingerprintProfile} [profile] one coherent network identity
|
|
88
|
+
* instead of a dozen knobs that can disagree — TLS, HTTP/2, header order and default headers
|
|
89
|
+
* together. Explicit options win over it. A profile that declares capabilities this package
|
|
90
|
+
* cannot perform is REFUSED rather than silently reduced: see `profiles.chrome`.
|
|
91
|
+
* @property {readonly string[]} [headerOrder] request header names, lowercased, in the order to
|
|
92
|
+
* emit them; `'*'` marks where headers not named go, in the order the caller gave them. Defaults
|
|
93
|
+
* to curl's (`CURL_HEADER_ORDER`). The platform `Headers` sorts alphabetically and lowercases, so
|
|
94
|
+
* without this a request goes out with `user-agent` last, which no real client does.
|
|
95
|
+
* @property {string[]} [http2PseudoHeaderOrder] request pseudo-headers in the order to emit them.
|
|
96
|
+
* Defaults to curl's. Any of the four omitted is appended rather than dropped: RFC 9113 s8.3.1
|
|
97
|
+
* makes all four mandatory, so a request missing one is malformed rather than merely unusual.
|
|
98
|
+
* @property {Record<string, 'incremental'|'without'|'never'>} [http2HpackIndexing] per-field HPACK
|
|
99
|
+
* indexing. Which fields enter the dynamic table is read by an Akamai-style h2 fingerprint.
|
|
100
|
+
* Defaults to curl's: everything incremental except `:path`.
|
|
101
|
+
* @property {Array<[number, number]>} [http2Settings] the HTTP/2 SETTINGS flight, as [id, value]
|
|
102
|
+
* pairs. Order is significant — an Akamai-style h2 fingerprint reads the ids in the order they
|
|
103
|
+
* are sent — so this replaces the flight rather than merging into it. Defaults to curl's. The
|
|
104
|
+
* TLS half of the fingerprint is configured through `tls` (`ciphers`, `groups`, `sigSchemes`,
|
|
105
|
+
* `alpn`, `versions`, `extensionOrder`).
|
|
85
106
|
* @property {boolean} [http2] offer HTTP/2 via ALPN and speak it when the server selects it.
|
|
86
107
|
* Default true. The goal is ACCESS, not speed — some sites treat HTTP/1.1 as a bot signal — and
|
|
87
108
|
* on a CPU-billed runtime h2 costs MORE than h1 (HPACK is extra work). Set false to offer only
|
|
@@ -101,7 +122,7 @@ export class Client {
|
|
|
101
122
|
* @param {ClientOptions} [options]
|
|
102
123
|
*/
|
|
103
124
|
constructor(options = {}) {
|
|
104
|
-
this.options = snapshotOptions(options);
|
|
125
|
+
this.options = snapshotOptions(applyProfile(options));
|
|
105
126
|
this.pool = new ConnectionPool(options.pool);
|
|
106
127
|
// HTTP/2 connections are NOT pooled the way h1 is: one connection multiplexes many concurrent
|
|
107
128
|
// streams, so it is not checked out per request. It lives here, keyed exactly like the h1 pool,
|
|
@@ -232,6 +253,9 @@ export function install(options = {}) {
|
|
|
232
253
|
|
|
233
254
|
async function performFetch(client, input, init) {
|
|
234
255
|
const o = client.options;
|
|
256
|
+
// Read the caller's header names and case before `Request` normalises them away: the platform
|
|
257
|
+
// `Headers` sorts lexicographically and lowercases, so by the line below both are already gone.
|
|
258
|
+
const callerOrder = callerHeaderOrder(input, init);
|
|
235
259
|
const request = new Request(input, init);
|
|
236
260
|
const redirectMode = init?.redirect ?? request.redirect ?? 'follow';
|
|
237
261
|
const maxRedirects = o.maxRedirects ?? DEFAULT_MAX_REDIRECTS;
|
|
@@ -246,7 +270,9 @@ async function performFetch(client, input, init) {
|
|
|
246
270
|
let current = {
|
|
247
271
|
url: new URL(request.url),
|
|
248
272
|
method: request.method,
|
|
249
|
-
|
|
273
|
+
// The caller's own list when it survived, so its order and case reach the wire; otherwise the
|
|
274
|
+
// normalised Headers, which is all there is left to work from.
|
|
275
|
+
headers: new OrderedHeaders(callerOrder ?? request.headers),
|
|
250
276
|
body,
|
|
251
277
|
};
|
|
252
278
|
const history = [];
|
|
@@ -436,6 +462,17 @@ function registerHttp2(client, key, conn) {
|
|
|
436
462
|
{ readable: conn.readable, writable: conn.writable, close: conn.close },
|
|
437
463
|
{
|
|
438
464
|
info: conn.info,
|
|
465
|
+
// The h2 half of the fingerprint. Passed through so a caller can present some client other
|
|
466
|
+
// than curl without reaching past the Client for it — the TLS half is configurable through
|
|
467
|
+
// `tls`, and one being reachable while the other was not made "the fingerprint is
|
|
468
|
+
// configurable" only half true.
|
|
469
|
+
...(client.options.http2Settings ? { settings: client.options.http2Settings } : {}),
|
|
470
|
+
...(client.options.http2PseudoHeaderOrder
|
|
471
|
+
? { pseudoHeaderOrder: client.options.http2PseudoHeaderOrder }
|
|
472
|
+
: {}),
|
|
473
|
+
...(client.options.http2HpackIndexing
|
|
474
|
+
? { hpackIndexing: client.options.http2HpackIndexing }
|
|
475
|
+
: {}),
|
|
439
476
|
onClose: () => {
|
|
440
477
|
client._h2conns.delete(h2);
|
|
441
478
|
// Only drop the keyed entry if it is still this connection; a newer one may have replaced it.
|
|
@@ -713,11 +750,15 @@ async function sendAndReceiveH2(client, h2, current, { deadlines }) {
|
|
|
713
750
|
*/
|
|
714
751
|
function buildH2Request(client, current, target) {
|
|
715
752
|
const o = client.options;
|
|
716
|
-
const headers = new
|
|
753
|
+
const headers = new OrderedHeaders(current.headers);
|
|
717
754
|
const defaultPort = current.url.protocol === 'https:' ? 443 : 80;
|
|
718
755
|
const authority =
|
|
719
756
|
target.port === defaultPort ? current.url.hostname : `${current.url.hostname}:${target.port}`;
|
|
720
757
|
|
|
758
|
+
// Profile headers are defaults: a request that sets its own User-Agent keeps it.
|
|
759
|
+
for (const [name, value] of client.options.profileHeaders ?? []) {
|
|
760
|
+
if (!headers.has(name)) headers.set(name, value);
|
|
761
|
+
}
|
|
721
762
|
if (!headers.has('accept')) headers.set('accept', '*/*');
|
|
722
763
|
if (!headers.has('accept-encoding') && o.decompress !== false) {
|
|
723
764
|
headers.set('accept-encoding', acceptEncodingFor(o.decoders));
|
|
@@ -737,12 +778,12 @@ function buildH2Request(client, current, target) {
|
|
|
737
778
|
else if (['POST', 'PUT', 'PATCH'].includes(current.method)) headers.set('content-length', '0');
|
|
738
779
|
else headers.delete('content-length');
|
|
739
780
|
|
|
740
|
-
//
|
|
741
|
-
//
|
|
742
|
-
//
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
return { authority, headers:
|
|
781
|
+
// Lowercased at the last moment, and only here: RFC 9113 s8.2.1 REQUIRES lowercase field names
|
|
782
|
+
// on an h2 wire and a server must treat an uppercase one as malformed. h1 is the opposite — real
|
|
783
|
+
// clients send `Host:`, not `host:` — which is why the case is carried this far and dropped only
|
|
784
|
+
// on this path. Pseudo-header order is fixed in http2/connection.js buildRequestFields.
|
|
785
|
+
headers.reorder(o.headerOrder ?? CURL_HEADER_ORDER);
|
|
786
|
+
return { authority, headers: headers.lowercased() };
|
|
746
787
|
}
|
|
747
788
|
|
|
748
789
|
function wantsKeepAlive(headInfo) {
|
|
@@ -793,13 +834,19 @@ function requestTarget(url) {
|
|
|
793
834
|
}
|
|
794
835
|
|
|
795
836
|
function buildHeaders(client, current, target) {
|
|
796
|
-
const headers = new
|
|
837
|
+
const headers = new OrderedHeaders(current.headers);
|
|
797
838
|
// Host is derived from the URL and never carried across a redirect; the default port is omitted
|
|
798
839
|
// because a server matching virtual hosts on the literal Host value expects it that way.
|
|
799
840
|
const defaultPort = current.url.protocol === 'https:' ? 443 : 80;
|
|
800
841
|
const hostValue =
|
|
801
842
|
target.port === defaultPort ? current.url.hostname : `${current.url.hostname}:${target.port}`;
|
|
802
843
|
|
|
844
|
+
// Profile headers are defaults: a request that sets its own User-Agent keeps it. Applied on both
|
|
845
|
+
// the h1 and h2 paths, which build their headers separately — the h2 side had this and the h1
|
|
846
|
+
// side did not, so a profile's User-Agent reached an h2 request and vanished from an h1 one.
|
|
847
|
+
for (const [name, value] of client.options.profileHeaders ?? []) {
|
|
848
|
+
if (!headers.has(name)) headers.set(name, value);
|
|
849
|
+
}
|
|
803
850
|
if (!headers.has('accept')) headers.set('accept', '*/*');
|
|
804
851
|
if (!headers.has('accept-encoding') && client.options.decompress !== false) {
|
|
805
852
|
// Never advertise br or zstd: the runtime has no DecompressionStream for either, so the
|
|
@@ -828,12 +875,11 @@ function buildHeaders(client, current, target) {
|
|
|
828
875
|
else if (['POST', 'PUT', 'PATCH'].includes(current.method)) headers.set('content-length', '0');
|
|
829
876
|
else headers.delete('content-length');
|
|
830
877
|
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
return ordered;
|
|
878
|
+
// Host is set rather than prepended so the order below decides where it goes — which for curl
|
|
879
|
+
// is first, but for a caller matching something else may not be.
|
|
880
|
+
headers.set('Host', hostValue);
|
|
881
|
+
headers.reorder(client.options.headerOrder ?? CURL_HEADER_ORDER);
|
|
882
|
+
return headers.entries();
|
|
837
883
|
}
|
|
838
884
|
|
|
839
885
|
export { CookieJar, ConnectionPool, utf8 };
|
package/src/http2/connection.js
CHANGED
|
@@ -117,6 +117,17 @@ export class Http2Retryable extends Http2Error {}
|
|
|
117
117
|
* @property {number} [maxConcurrentStreams] our advertised SETTINGS_MAX_CONCURRENT_STREAMS.
|
|
118
118
|
* @property {number} [maxHeaderTableSize] our advertised SETTINGS_HEADER_TABLE_SIZE.
|
|
119
119
|
* @property {number} [maxHeaderListSize] self-protection cap on a decoded response header list.
|
|
120
|
+
* @property {string[]} [pseudoHeaderOrder] request pseudo-headers in the order to emit them.
|
|
121
|
+
* Defaults to curl's `[':method', ':scheme', ':authority', ':path']`. Any of the four left out is
|
|
122
|
+
* appended rather than dropped — RFC 9113 s8.3.1 makes all four mandatory and a request missing
|
|
123
|
+
* one is malformed, which is not a fingerprint choice anyone should be able to make by accident.
|
|
124
|
+
* @property {Record<string, 'incremental'|'without'|'never'>} [hpackIndexing] per-field HPACK
|
|
125
|
+
* indexing. Which fields enter the dynamic table is part of the fingerprint. Defaults to curl's:
|
|
126
|
+
* everything incremental except `:path`, which is sent without indexing.
|
|
127
|
+
* @property {Array<[number, number]>} [settings] the SETTINGS flight sent in the connection
|
|
128
|
+
* preface, as [id, value] pairs. Order is significant — an Akamai-style HTTP/2 fingerprint reads
|
|
129
|
+
* the ids in the order they are sent — so this replaces the flight entirely rather than merging.
|
|
130
|
+
* Defaults to curl's: MAX_CONCURRENT_STREAMS, INITIAL_WINDOW_SIZE, ENABLE_PUSH.
|
|
120
131
|
* @property {number} [maxHeaderBlockBytes] cap on the RAW bytes of one HEADERS+CONTINUATION run,
|
|
121
132
|
* before HPACK decoding. Default 262144, matching the decoded cap. This is the bound that stops
|
|
122
133
|
* a CONTINUATION flood; `maxHeaderListSize` cannot, because it is only reachable once the whole
|
|
@@ -196,6 +207,12 @@ export class Http2Connection {
|
|
|
196
207
|
// reference makes a small input decode LARGER, never the reverse — so a block whose raw size
|
|
197
208
|
// exceeds the decoded cap could not have produced an acceptable header list anyway.
|
|
198
209
|
this._maxHeaderBlockBytes = opts.maxHeaderBlockBytes ?? 262144;
|
|
210
|
+
/** @type {Array<[number, number]> | null} the SETTINGS flight, ids and order included */
|
|
211
|
+
this._settingsFlight = opts.settings ?? null;
|
|
212
|
+
// The rest of what an Akamai-style h2 fingerprint reads: the pseudo-header order and which
|
|
213
|
+
// fields go into the HPACK dynamic table. Both default to curl's, both captured off the wire.
|
|
214
|
+
this._pseudoHeaderOrder = opts.pseudoHeaderOrder ?? null;
|
|
215
|
+
this._hpackIndexing = opts.hpackIndexing ?? null;
|
|
199
216
|
this._expectFirstSettings = true;
|
|
200
217
|
|
|
201
218
|
this._fatal = null; // set once; rejects every stream and every future request
|
|
@@ -230,11 +247,19 @@ export class Http2Connection {
|
|
|
230
247
|
_sendPreface() {
|
|
231
248
|
// Exactly curl's flight and order: the 24-byte magic, then SETTINGS (ids 3,4,2), then a
|
|
232
249
|
// connection-level WINDOW_UPDATE that raises the receive window to 1000 MiB. See constants.js.
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
250
|
+
//
|
|
251
|
+
// The ORDER of the settings, not just their values, is part of the fingerprint an Akamai-style
|
|
252
|
+
// h2 hash reads, so a caller matching some other client needs to be able to set both. Supplying
|
|
253
|
+
// `settings` replaces the flight wholesale; the values still drive this connection's own
|
|
254
|
+
// behaviour, so a caller who advertises a window it will not honour has misconfigured the
|
|
255
|
+
// connection rather than merely disguised it.
|
|
256
|
+
const settings = settingsFrame(
|
|
257
|
+
this._settingsFlight ?? [
|
|
258
|
+
[SETTINGS.MAX_CONCURRENT_STREAMS, this._ourMaxConcurrent],
|
|
259
|
+
[SETTINGS.INITIAL_WINDOW_SIZE, this._ourInitialWindow],
|
|
260
|
+
[SETTINGS.ENABLE_PUSH, 0],
|
|
261
|
+
],
|
|
262
|
+
);
|
|
238
263
|
const inc = this._ourConnWindow - DEFAULT_INITIAL_WINDOW;
|
|
239
264
|
const flight =
|
|
240
265
|
inc > 0
|
|
@@ -307,7 +332,10 @@ export class Http2Connection {
|
|
|
307
332
|
}
|
|
308
333
|
const hasBody = body != null && body.byteLength > 0;
|
|
309
334
|
|
|
310
|
-
const fields = buildRequestFields(
|
|
335
|
+
const fields = buildRequestFields(
|
|
336
|
+
{ method, scheme, authority, path, headers },
|
|
337
|
+
{ pseudoHeaderOrder: this._pseudoHeaderOrder, hpackIndexing: this._hpackIndexing },
|
|
338
|
+
);
|
|
311
339
|
const block = encodeHeaderBlock(fields);
|
|
312
340
|
this._sendHeaderBlock(id, block, !hasBody);
|
|
313
341
|
stream.localEnded = !hasBody;
|
|
@@ -1192,14 +1220,32 @@ export class Http2Connection {
|
|
|
1192
1220
|
* headers: Array<[string, string]> }} req
|
|
1193
1221
|
* @returns {import('./hpack.js').HpackField[]}
|
|
1194
1222
|
*/
|
|
1195
|
-
export function buildRequestFields({ method, scheme, authority, path, headers }) {
|
|
1223
|
+
export function buildRequestFields({ method, scheme, authority, path, headers }, opts = {}) {
|
|
1196
1224
|
const pseudo = { ':method': method, ':scheme': scheme, ':authority': authority, ':path': path };
|
|
1225
|
+
const order = opts.pseudoHeaderOrder ?? PSEUDO_HEADER_ORDER;
|
|
1226
|
+
// Which fields go into the dynamic table is itself part of the fingerprint: an Akamai-style h2
|
|
1227
|
+
// hash reads the HPACK representation, and curl indexes everything except :path. A caller
|
|
1228
|
+
// matching another client needs both this and the order, so both are configurable — with the
|
|
1229
|
+
// caller's map consulted first and curl's rule as the default.
|
|
1230
|
+
const indexingFor = (name) =>
|
|
1231
|
+
opts.hpackIndexing?.[name] ?? (name === ':path' ? 'without' : 'incremental');
|
|
1232
|
+
|
|
1197
1233
|
const fields = [];
|
|
1234
|
+
const seen = new Set();
|
|
1235
|
+
for (const name of order) {
|
|
1236
|
+
if (!(name in pseudo) || seen.has(name)) continue;
|
|
1237
|
+
seen.add(name);
|
|
1238
|
+
fields.push({ name, value: pseudo[name], indexing: indexingFor(name) });
|
|
1239
|
+
}
|
|
1240
|
+
// A caller-supplied order that omits a pseudo-header would produce a malformed request
|
|
1241
|
+
// (RFC 9113 s8.3.1 makes all four mandatory for a request), so the missing ones are appended in
|
|
1242
|
+
// curl's order rather than silently dropped.
|
|
1198
1243
|
for (const name of PSEUDO_HEADER_ORDER) {
|
|
1199
|
-
|
|
1244
|
+
if (seen.has(name)) continue;
|
|
1245
|
+
fields.push({ name, value: pseudo[name], indexing: indexingFor(name) });
|
|
1200
1246
|
}
|
|
1201
1247
|
for (const [name, value] of headers) {
|
|
1202
|
-
fields.push({ name, value, indexing:
|
|
1248
|
+
fields.push({ name, value, indexing: indexingFor(name) });
|
|
1203
1249
|
}
|
|
1204
1250
|
return fields;
|
|
1205
1251
|
}
|
package/src/index.js
CHANGED