tunnelfetch 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +28 -0
- package/README.md +617 -0
- package/README.zh-CN.md +470 -0
- package/package.json +74 -0
- package/src/client/cookies.js +429 -0
- package/src/client/decode.js +346 -0
- package/src/client/redirect.js +249 -0
- package/src/client.js +704 -0
- package/src/errors.js +181 -0
- package/src/http1/chunked.js +289 -0
- package/src/http1/index.js +10 -0
- package/src/http1/request.js +143 -0
- package/src/http1/response.js +493 -0
- package/src/http2/connection.js +1170 -0
- package/src/http2/constants.js +129 -0
- package/src/http2/frames.js +291 -0
- package/src/http2/hpack.js +420 -0
- package/src/http2/huffman.js +203 -0
- package/src/http2/index.js +21 -0
- package/src/index.js +46 -0
- package/src/pool.js +256 -0
- package/src/proxy/direct.js +62 -0
- package/src/proxy/http-connect.js +206 -0
- package/src/proxy/index.js +197 -0
- package/src/proxy/socks5.js +344 -0
- package/src/tls/aead.js +263 -0
- package/src/tls/connect.js +407 -0
- package/src/tls/constants.js +334 -0
- package/src/tls/extensions.js +376 -0
- package/src/tls/handshake-messages.js +901 -0
- package/src/tls/handshake.js +568 -0
- package/src/tls/handshake12.js +507 -0
- package/src/tls/index.js +44 -0
- package/src/tls/keyschedule.js +473 -0
- package/src/tls/record.js +872 -0
- package/src/tls/tickets.js +145 -0
- package/src/tls/transcript.js +101 -0
- package/src/tls/wire.js +224 -0
- package/src/transport.js +296 -0
- package/src/trust/der.js +551 -0
- package/src/trust/index.js +375 -0
- package/src/trust/name.js +235 -0
- package/src/trust/ocsp.js +759 -0
- package/src/trust/path.js +595 -0
- package/src/trust/roots.js +454 -0
- package/src/trust/x509.js +902 -0
- package/src/util/bytes.js +470 -0
- package/src/util/deadline.js +266 -0
- package/src/warmup-fixture.js +85 -0
- package/src/warmup.js +243 -0
- package/types/client/cookies.d.ts +159 -0
- package/types/client/decode.d.ts +54 -0
- package/types/client/redirect.d.ts +96 -0
- package/types/client.d.ts +323 -0
- package/types/errors.d.ts +141 -0
- package/types/http1/chunked.d.ts +48 -0
- package/types/http1/index.d.ts +3 -0
- package/types/http1/request.d.ts +44 -0
- package/types/http1/response.d.ts +183 -0
- package/types/http2/connection.d.ts +282 -0
- package/types/http2/constants.d.ts +95 -0
- package/types/http2/frames.d.ts +116 -0
- package/types/http2/hpack.d.ts +99 -0
- package/types/http2/huffman.d.ts +21 -0
- package/types/http2/index.d.ts +5 -0
- package/types/index.d.ts +17 -0
- package/types/pool.d.ts +135 -0
- package/types/proxy/direct.d.ts +26 -0
- package/types/proxy/http-connect.d.ts +37 -0
- package/types/proxy/index.d.ts +62 -0
- package/types/proxy/socks5.d.ts +47 -0
- package/types/tls/aead.d.ts +67 -0
- package/types/tls/connect.d.ts +280 -0
- package/types/tls/constants.d.ts +275 -0
- package/types/tls/extensions.d.ts +195 -0
- package/types/tls/handshake-messages.d.ts +430 -0
- package/types/tls/handshake.d.ts +90 -0
- package/types/tls/handshake12.d.ts +35 -0
- package/types/tls/index.d.ts +9 -0
- package/types/tls/keyschedule.d.ts +272 -0
- package/types/tls/record.d.ts +361 -0
- package/types/tls/tickets.d.ts +66 -0
- package/types/tls/transcript.d.ts +52 -0
- package/types/tls/wire.d.ts +106 -0
- package/types/transport.d.ts +222 -0
- package/types/trust/der.d.ts +239 -0
- package/types/trust/index.d.ts +194 -0
- package/types/trust/name.d.ts +33 -0
- package/types/trust/ocsp.d.ts +138 -0
- package/types/trust/path.d.ts +139 -0
- package/types/trust/roots.d.ts +36 -0
- package/types/trust/x509.d.ts +401 -0
- package/types/util/bytes.d.ts +183 -0
- package/types/util/deadline.d.ts +133 -0
- package/types/warmup-fixture.d.ts +11 -0
- package/types/warmup.d.ts +45 -0
package/src/client.js
ADDED
|
@@ -0,0 +1,704 @@
|
|
|
1
|
+
// The fetch-shaped facade.
|
|
2
|
+
//
|
|
3
|
+
// The public surface is deliberately the platform's own: requests come in as `Request`, responses
|
|
4
|
+
// go out as `Response`. That is not decoration — the OpenAI and Anthropic SDKs, and most of the
|
|
5
|
+
// libraries anyone would want to route through a proxy, accept a custom `fetch` and nothing else.
|
|
6
|
+
// Being that function is the deliverable.
|
|
7
|
+
//
|
|
8
|
+
// Delegation to the platform's fetch is a capability decision, not a convenience one; see
|
|
9
|
+
// `nativeFetchCanServe` in transport.js for why. Nothing here ever installs itself: replacing a
|
|
10
|
+
// global at import time turns every unrelated failure in a process into a mystery about this
|
|
11
|
+
// package, so `install()` exists, is explicit, and hands back its own undo.
|
|
12
|
+
|
|
13
|
+
import { ConfigError, HttpError, TunnelFetchError, codes } from './errors.js';
|
|
14
|
+
import { ByteReader, ByteWriter, UnexpectedEofError, concat, utf8 } from './util/bytes.js';
|
|
15
|
+
import { serializeRequestHead } from './http1/request.js';
|
|
16
|
+
import { bodyFraming, readResponseBody, readResponseHead } from './http1/response.js';
|
|
17
|
+
import { ACCEPT_ENCODING, decodeBody } from './client/decode.js';
|
|
18
|
+
import { CookieJar } from './client/cookies.js';
|
|
19
|
+
import { DEFAULT_MAX_REDIRECTS, nextRequest, shouldRedirect } from './client/redirect.js';
|
|
20
|
+
import { ConnectionPool, poolKey } from './pool.js';
|
|
21
|
+
import { TicketStore } from './tls/tickets.js';
|
|
22
|
+
import { DeadlineController, withIdleDeadline } from './util/deadline.js';
|
|
23
|
+
import { nativeFetchCanServe, openConnection, targetFromUrl } from './transport.js';
|
|
24
|
+
import { parseProxy } from './proxy/index.js';
|
|
25
|
+
import { Http2Connection, Http2Retryable } from './http2/connection.js';
|
|
26
|
+
import { ALPN_H2, ALPN_HTTP11 } from './http2/constants.js';
|
|
27
|
+
|
|
28
|
+
/** Status codes whose Response may not carry a body, per the Response constructor. */
|
|
29
|
+
const NULL_BODY_STATUS = new Set([101, 204, 205, 304]);
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* A `fetch`-shaped function. Deliberately the platform's own signature: being assignable to
|
|
33
|
+
* `typeof fetch` is what lets an SDK accept this in place of the global without adapting.
|
|
34
|
+
* @typedef {(input: RequestInfo | URL, init?: RequestInit) => Promise<Response>} FetchLike
|
|
35
|
+
*/
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* What a Response carries about the connection that produced it, under the non-standard
|
|
39
|
+
* `tunnelfetch` property.
|
|
40
|
+
* @typedef {object} ResponseDetail
|
|
41
|
+
* @property {string} url
|
|
42
|
+
* @property {boolean} proxied
|
|
43
|
+
* @property {string | null} proxy the proxy actually used, credentials omitted
|
|
44
|
+
* @property {import('./tls/connect.js').TlsSessionInfo | null} tls null for cleartext
|
|
45
|
+
* @property {'1.0' | '1.1' | '2'} httpVersion '2' when ALPN negotiated HTTP/2
|
|
46
|
+
* @property {'none' | 'content-length' | 'chunked' | 'until-close' | 'h2'} framing 'h2' when the
|
|
47
|
+
* body was delimited by an HTTP/2 END_STREAM rather than by any HTTP/1.1 framing rule
|
|
48
|
+
*/
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Limits on what a peer may make us buffer. Each is a fail-closed cap, not a hint.
|
|
52
|
+
* @typedef {object} Limits
|
|
53
|
+
* @property {number} [maxHeaderBytes] response head, default 65536
|
|
54
|
+
* @property {number} [maxProxyReplyBytes] proxy CONNECT reply head, default 32768
|
|
55
|
+
*/
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* @typedef {object} ClientOptions
|
|
59
|
+
* @property {import('./proxy/index.js').ConnectFn} [connect] Socket factory. Required for any
|
|
60
|
+
* request the platform's own `fetch` cannot serve — which is every proxied request, and every
|
|
61
|
+
* request asking for a trust policy `fetch` cannot express.
|
|
62
|
+
* @property {string | import('./proxy/index.js').ProxyConfig | null} [proxy] URL string or
|
|
63
|
+
* config object; `http:`, `https:`, `socks5:` and `socks5h:`.
|
|
64
|
+
* @property {import('./trust/index.js').TrustConfig} [trust] certificate policy, httpx's
|
|
65
|
+
* `verify=`. Default `{ mode: 'system' }`.
|
|
66
|
+
* @property {import('./tls/connect.js').TlsOptions} [tls] handshake knobs.
|
|
67
|
+
* @property {import('./util/deadline.js').DeadlineOptions} [timeouts] connect / handshake /
|
|
68
|
+
* headers / idle / total, in ms. The idle gap is the control; total is a backstop.
|
|
69
|
+
* @property {boolean} [cookies] enable a per-Client cookie jar.
|
|
70
|
+
* @property {import('./client/cookies.js').CookieJar} [jar] supply a jar directly, e.g. to share
|
|
71
|
+
* one across Clients or to persist it.
|
|
72
|
+
* @property {number} [maxRedirects] default 20.
|
|
73
|
+
* @property {number} [maxBodyBytes] enforced from Content-Length before a byte is read.
|
|
74
|
+
* @property {boolean} [decompress] gzip/deflate. Default true. Never `br`; the runtime cannot
|
|
75
|
+
* decompress it, so it is never advertised either.
|
|
76
|
+
* @property {boolean} [keepAlive] default true.
|
|
77
|
+
* @property {boolean} [http2] offer HTTP/2 via ALPN and speak it when the server selects it.
|
|
78
|
+
* Default true. The goal is ACCESS, not speed — some sites treat HTTP/1.1 as a bot signal — and
|
|
79
|
+
* on a CPU-billed runtime h2 costs MORE than h1 (HPACK is extra work). Set false to offer only
|
|
80
|
+
* `http/1.1`. There is no fallback-and-retry either way: the server's ALPN pick is followed.
|
|
81
|
+
* @property {boolean} [forceTunnel] never delegate to the platform's fetch, even when it could
|
|
82
|
+
* serve the request. Mainly for exercising this stack against origins that do not need it.
|
|
83
|
+
* @property {FetchLike} [nativeFetch] delegation target; defaults to `globalThis.fetch`.
|
|
84
|
+
* @property {{ maxPerKey?: number, maxTotal?: number }} [pool] connection pool sizing.
|
|
85
|
+
* @property {Limits} [limits]
|
|
86
|
+
* @property {import('./tls/connect.js').TlsDeps} [deps] injectable randomness and key generation.
|
|
87
|
+
* @property {AbortSignal} [signal] aborts every request this Client makes.
|
|
88
|
+
* @property {number} [now] epoch ms override, for certificate validity in tests.
|
|
89
|
+
*/
|
|
90
|
+
|
|
91
|
+
export class Client {
|
|
92
|
+
/**
|
|
93
|
+
* @param {ClientOptions} [options]
|
|
94
|
+
*/
|
|
95
|
+
constructor(options = {}) {
|
|
96
|
+
this.options = { ...options };
|
|
97
|
+
this.pool = new ConnectionPool(options.pool);
|
|
98
|
+
// HTTP/2 connections are NOT pooled the way h1 is: one connection multiplexes many concurrent
|
|
99
|
+
// streams, so it is not checked out per request. It lives here, keyed exactly like the h1 pool,
|
|
100
|
+
// and is shared until it goes away. See http2/connection.js for why the exclusive-checkout
|
|
101
|
+
// model does not apply. `Map<poolKey, Http2Connection>`.
|
|
102
|
+
/** @type {Map<string, import('./http2/connection.js').Http2Connection>} */
|
|
103
|
+
this._h2 = new Map();
|
|
104
|
+
// Every live h2 connection, keyed map or not. The map holds the one CURRENTLY dispatchable
|
|
105
|
+
// connection per key; if two first-requests to a new key race, the loser is orphaned from the
|
|
106
|
+
// map but is still open and serving its stream, so it must be tracked here or close() would
|
|
107
|
+
// leak it. Entries remove themselves on death.
|
|
108
|
+
/** @type {Set<import('./http2/connection.js').Http2Connection>} */
|
|
109
|
+
this._h2conns = new Set();
|
|
110
|
+
this.jar = options.cookies ? (options.jar ?? new CookieJar()) : (options.jar ?? null);
|
|
111
|
+
// TLS 1.3 session tickets, per-Client for the same reason the pool is: a ticket is a
|
|
112
|
+
// credential bound to a trust configuration, and it is stored and looked up under the SAME
|
|
113
|
+
// key the pool uses, so a ticket can never resume a session for a request whose scheme,
|
|
114
|
+
// host, port, proxy, trust policy or TLS options differ from the connection that earned it.
|
|
115
|
+
// `options.now` (the test override for certificate validity) drives ticket ages too — the
|
|
116
|
+
// two are the same clock question.
|
|
117
|
+
this.tickets = new TicketStore(
|
|
118
|
+
typeof options.now === 'number' ? { now: () => options.now } : {},
|
|
119
|
+
);
|
|
120
|
+
this._closed = false;
|
|
121
|
+
// Bound so a Client can be handed straight to an SDK expecting a bare function.
|
|
122
|
+
this.fetch = this.fetch.bind(this);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* @param {RequestInfo | URL} input
|
|
127
|
+
* @param {RequestInit} [init]
|
|
128
|
+
* @returns {Promise<Response & { tunnelfetch?: ResponseDetail }>}
|
|
129
|
+
*/
|
|
130
|
+
async fetch(input, init) {
|
|
131
|
+
if (this._closed) {
|
|
132
|
+
throw new TunnelFetchError(codes.POOL_CLOSED, 'this Client has been closed');
|
|
133
|
+
}
|
|
134
|
+
return performFetch(this, input, init);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/** Release every pooled socket and shared HTTP/2 connection. A Client that is finished must be
|
|
138
|
+
* closed or sockets leak for the isolate's lifetime. */
|
|
139
|
+
async close() {
|
|
140
|
+
this._closed = true;
|
|
141
|
+
this.tickets.clear(); // tickets are credentials; a closed Client keeps none
|
|
142
|
+
const h2 = [...this._h2conns]; // the set, not the map: it also holds race-orphaned connections
|
|
143
|
+
this._h2.clear();
|
|
144
|
+
this._h2conns.clear();
|
|
145
|
+
await Promise.all([this.pool.closeAll(), ...h2.map((c) => c.close().catch(() => {}))]);
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* A standalone fetch bound to a configuration, matching httpx's module-level helpers. Creates and
|
|
151
|
+
* closes a Client per call, so no connection is reused; use `new Client()` when reuse matters.
|
|
152
|
+
*
|
|
153
|
+
* @param {ClientOptions} [options]
|
|
154
|
+
* @returns {FetchLike}
|
|
155
|
+
*/
|
|
156
|
+
export function createFetch(options = {}) {
|
|
157
|
+
return async function tunnelFetch(input, init) {
|
|
158
|
+
const client = new Client(options);
|
|
159
|
+
try {
|
|
160
|
+
return await client.fetch(input, init);
|
|
161
|
+
} finally {
|
|
162
|
+
await client.close();
|
|
163
|
+
}
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Replace `globalThis.fetch`, for libraries that only accept the global. Returns the undo.
|
|
169
|
+
* Never called automatically, and never on import.
|
|
170
|
+
*
|
|
171
|
+
* @param {ClientOptions} [options]
|
|
172
|
+
* @returns {() => void} uninstall; idempotent, and a no-op if someone else has since taken the global
|
|
173
|
+
*/
|
|
174
|
+
export function install(options = {}) {
|
|
175
|
+
const previous = globalThis.fetch;
|
|
176
|
+
const replacement = createFetch(options);
|
|
177
|
+
globalThis.fetch = replacement;
|
|
178
|
+
let undone = false;
|
|
179
|
+
return function uninstall() {
|
|
180
|
+
if (undone) return;
|
|
181
|
+
undone = true;
|
|
182
|
+
// Only restore if nobody replaced us in the meantime; clobbering a third party's global on
|
|
183
|
+
// the way out would be the same mistake in reverse.
|
|
184
|
+
if (globalThis.fetch === replacement) globalThis.fetch = previous;
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
// ------------------------------------------------------------------ the request loop
|
|
189
|
+
|
|
190
|
+
async function performFetch(client, input, init) {
|
|
191
|
+
const o = client.options;
|
|
192
|
+
const request = new Request(input, init);
|
|
193
|
+
const redirectMode = init?.redirect ?? request.redirect ?? 'follow';
|
|
194
|
+
const maxRedirects = o.maxRedirects ?? DEFAULT_MAX_REDIRECTS;
|
|
195
|
+
|
|
196
|
+
// Buffer the body once so a 307/308 can replay it. A stream body cannot be replayed, and
|
|
197
|
+
// pretending otherwise by sending an empty body on the second hop would corrupt the request.
|
|
198
|
+
let body = null;
|
|
199
|
+
if (request.body) {
|
|
200
|
+
body = new Uint8Array(await request.arrayBuffer());
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
let current = {
|
|
204
|
+
url: new URL(request.url),
|
|
205
|
+
method: request.method,
|
|
206
|
+
headers: new Headers(request.headers),
|
|
207
|
+
body,
|
|
208
|
+
};
|
|
209
|
+
const history = [];
|
|
210
|
+
|
|
211
|
+
for (let hop = 0; ; hop++) {
|
|
212
|
+
const response = await exchange(client, current, { hop });
|
|
213
|
+
|
|
214
|
+
if (redirectMode !== 'follow' || !shouldRedirect(response.status, current.method)) {
|
|
215
|
+
if (redirectMode === 'error' && shouldRedirect(response.status, current.method)) {
|
|
216
|
+
throw new HttpError(
|
|
217
|
+
codes.REDIRECT_SCHEME,
|
|
218
|
+
`redirect mode is "error" but the server answered ${response.status}`,
|
|
219
|
+
{ status: response.status },
|
|
220
|
+
);
|
|
221
|
+
}
|
|
222
|
+
return finish(response, current.url, history.length > 0);
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
// nextRequest owns `history`: it appends this hop's key (for loop detection) and reads the
|
|
226
|
+
// array's length as the hop budget. Pushing here as well would count every hop twice and halve
|
|
227
|
+
// the effective maxRedirects — the `redirected` flag below reads the same array and stays
|
|
228
|
+
// correct because nextRequest appends exactly one entry per redirect followed.
|
|
229
|
+
const next = nextRequest(current, response, { maxRedirects, history });
|
|
230
|
+
// The previous body must be drained (or discarded) before the socket can be reused, and the
|
|
231
|
+
// caller will never read a redirect's body.
|
|
232
|
+
await response.body?.cancel?.().catch(() => {});
|
|
233
|
+
current = next;
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
function finish(response, url, redirected) {
|
|
238
|
+
// `url` and `redirected` are read-only on Response, but every consumer expects them to reflect
|
|
239
|
+
// where the body actually came from.
|
|
240
|
+
Object.defineProperty(response, 'url', { value: url.href, configurable: true });
|
|
241
|
+
Object.defineProperty(response, 'redirected', { value: redirected, configurable: true });
|
|
242
|
+
return response;
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
// ------------------------------------------------------------------ one request/response
|
|
246
|
+
|
|
247
|
+
async function exchange(client, current, { hop }) {
|
|
248
|
+
const o = client.options;
|
|
249
|
+
const proxy = parseProxy(o.proxy ?? null);
|
|
250
|
+
const trust = o.trust ?? { mode: 'system' };
|
|
251
|
+
const tls = o.tls ?? {};
|
|
252
|
+
|
|
253
|
+
const native = nativeFetchCanServe({ proxy, trust, tls, forceTunnel: o.forceTunnel });
|
|
254
|
+
if (native.ok) {
|
|
255
|
+
const fetchImpl = o.nativeFetch ?? globalThis.fetch;
|
|
256
|
+
if (typeof fetchImpl === 'function') {
|
|
257
|
+
return fetchImpl(current.url.href, {
|
|
258
|
+
method: current.method,
|
|
259
|
+
headers: current.headers,
|
|
260
|
+
body: current.body ?? undefined,
|
|
261
|
+
redirect: 'manual',
|
|
262
|
+
});
|
|
263
|
+
}
|
|
264
|
+
} else if (typeof o.connect !== 'function') {
|
|
265
|
+
throw new ConfigError(
|
|
266
|
+
codes.CONFIG_UNSATISFIABLE,
|
|
267
|
+
`this request cannot use the platform's fetch (${native.reason}) and no connect function ` +
|
|
268
|
+
'was configured. Supply `connect`: the raw-TCP socket factory of the host runtime, ' +
|
|
269
|
+
'which must return { readable, writable, opened?, close? }. The README names the exact ' +
|
|
270
|
+
'import for each supported runtime.',
|
|
271
|
+
{ reason: native.reason },
|
|
272
|
+
);
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
const target = targetFromUrl(current.url);
|
|
276
|
+
const key = poolKey({
|
|
277
|
+
scheme: current.url.protocol,
|
|
278
|
+
hostname: target.hostname,
|
|
279
|
+
port: target.port,
|
|
280
|
+
proxy,
|
|
281
|
+
trust,
|
|
282
|
+
tls,
|
|
283
|
+
});
|
|
284
|
+
|
|
285
|
+
// 1. An existing shared HTTP/2 connection for this key multiplexes this request as a new stream.
|
|
286
|
+
const shared = client._h2.get(key);
|
|
287
|
+
if (shared && shared.canDispatch()) {
|
|
288
|
+
const deadlines = new DeadlineController(o.timeouts ?? {}, { signal: o.signal });
|
|
289
|
+
try {
|
|
290
|
+
return await sendAndReceiveH2(client, shared, current, { deadlines });
|
|
291
|
+
} catch (err) {
|
|
292
|
+
deadlines.dispose();
|
|
293
|
+
// A provably-unprocessed stream (GOAWAY past it, REFUSED_STREAM) is the h2 analogue of h1's
|
|
294
|
+
// "server never saw it": safe to re-send on a fresh connection. Anything else propagates.
|
|
295
|
+
if (!(err instanceof Http2Retryable)) throw err;
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
// 2. A pooled HTTP/1.1 connection, taken exclusively for this one request.
|
|
300
|
+
const pooled = client.pool.take(key);
|
|
301
|
+
if (pooled) {
|
|
302
|
+
const deadlines = new DeadlineController(o.timeouts ?? {}, { signal: o.signal });
|
|
303
|
+
try {
|
|
304
|
+
return await sendAndReceive(client, pooled, current, { key, deadlines, reused: true, hop });
|
|
305
|
+
} catch (err) {
|
|
306
|
+
deadlines.dispose();
|
|
307
|
+
await client.pool.discard(pooled);
|
|
308
|
+
throw err;
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
// 3. A fresh connection. Its negotiated ALPN — not a guess, not a retry — decides h2 or h1.
|
|
313
|
+
return openFreshAndSend(client, current, { hop, key, proxy, trust, tls });
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
/**
|
|
317
|
+
* Open a new connection for `key` and dispatch the request over whichever protocol ALPN selected.
|
|
318
|
+
* There is no reconnect-and-retry-lower anywhere: the server picked, and we speak that.
|
|
319
|
+
*/
|
|
320
|
+
async function openFreshAndSend(client, current, { hop, key, proxy, trust, tls }) {
|
|
321
|
+
const o = client.options;
|
|
322
|
+
const deadlines = new DeadlineController(o.timeouts ?? {}, { signal: o.signal });
|
|
323
|
+
// Offer h2 unless the caller disabled it or pinned their own ALPN list. Newest first: ALPN is a
|
|
324
|
+
// client preference list and the server chooses from it.
|
|
325
|
+
const alpn = tls.alpn ?? (o.http2 === false ? [ALPN_HTTP11] : [ALPN_H2, ALPN_HTTP11]);
|
|
326
|
+
let conn;
|
|
327
|
+
let isH2 = false;
|
|
328
|
+
try {
|
|
329
|
+
conn = await openConnection({
|
|
330
|
+
url: current.url,
|
|
331
|
+
connect: o.connect,
|
|
332
|
+
proxy,
|
|
333
|
+
trust,
|
|
334
|
+
tls,
|
|
335
|
+
alpn,
|
|
336
|
+
resumption: resumptionFor(client, key),
|
|
337
|
+
deps: o.deps,
|
|
338
|
+
deadlines,
|
|
339
|
+
limits: o.limits ?? {},
|
|
340
|
+
now: o.now,
|
|
341
|
+
});
|
|
342
|
+
if (conn.info.tls?.alpnProtocol === ALPN_H2) {
|
|
343
|
+
isH2 = true;
|
|
344
|
+
const h2 = registerHttp2(client, key, conn);
|
|
345
|
+
return await sendAndReceiveH2(client, h2, current, { deadlines });
|
|
346
|
+
}
|
|
347
|
+
return await sendAndReceive(client, conn, current, { key, deadlines, reused: false, hop });
|
|
348
|
+
} catch (err) {
|
|
349
|
+
deadlines.dispose();
|
|
350
|
+
// An h2 connection owns its own duplex and deregisters itself on death (registerHttp2's
|
|
351
|
+
// onClose); a stream-level failure leaves it healthy and registered for reuse, so it must not
|
|
352
|
+
// be discarded here. Only the h1 socket is the pool's to throw away.
|
|
353
|
+
if (conn && !isH2) await client.pool.discard(conn);
|
|
354
|
+
throw err;
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
/**
|
|
359
|
+
* Session-resumption wiring for one connection attempt: the freshest usable ticket stored under
|
|
360
|
+
* this pool key (consumed now, single-use), and the capture callback that files new tickets
|
|
361
|
+
* under the same key. That the offer and the capture share the connection's own pool key is the
|
|
362
|
+
* entire safety argument — a ticket can only ever be replayed to the exact
|
|
363
|
+
* scheme/host/port/proxy/trust/tls tuple that earned it.
|
|
364
|
+
*
|
|
365
|
+
* One policy carve-out: under `revocation: 'require-staple'` resumption is disabled outright
|
|
366
|
+
* (nothing stored, nothing offered). That caller demanded a stapled revocation proof on every
|
|
367
|
+
* connection, and a resumed handshake carries no certificate and therefore no staple — resuming
|
|
368
|
+
* would silently skip the exact check that was made mandatory. Costs one full handshake per
|
|
369
|
+
* connection, which is precisely what that trust posture asks for.
|
|
370
|
+
*
|
|
371
|
+
* @param {Client} client
|
|
372
|
+
* @param {string} key
|
|
373
|
+
* @returns {{ offer: object | null, onTicket: (t: object) => void } | undefined}
|
|
374
|
+
*/
|
|
375
|
+
function resumptionFor(client, key) {
|
|
376
|
+
const trust = client.options.trust ?? { mode: 'system' };
|
|
377
|
+
if (trust.revocation === 'require-staple') return undefined;
|
|
378
|
+
// Keyed by the FULL pool key, which already folds scheme, host, port, proxy, trust policy and
|
|
379
|
+
// TLS options — for exactly the reason the pool does it. A ticket is a credential: one obtained
|
|
380
|
+
// under a pinned or custom trust policy must never resume a connection for a caller who asked
|
|
381
|
+
// for something else, and keying by origin alone is precisely how that happens.
|
|
382
|
+
return {
|
|
383
|
+
offer: client.tickets.take(key),
|
|
384
|
+
onTicket: (t) => {
|
|
385
|
+
if (!client._closed) client.tickets.put(key, t);
|
|
386
|
+
},
|
|
387
|
+
};
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
/** Wrap a freshly negotiated h2 connection, register it for reuse, and arrange its own removal. */
|
|
391
|
+
function registerHttp2(client, key, conn) {
|
|
392
|
+
const h2 = new Http2Connection(
|
|
393
|
+
{ readable: conn.readable, writable: conn.writable, close: conn.close },
|
|
394
|
+
{
|
|
395
|
+
info: conn.info,
|
|
396
|
+
onClose: () => {
|
|
397
|
+
client._h2conns.delete(h2);
|
|
398
|
+
// Only drop the keyed entry if it is still this connection; a newer one may have replaced it.
|
|
399
|
+
if (client._h2.get(key) === h2) client._h2.delete(key);
|
|
400
|
+
},
|
|
401
|
+
},
|
|
402
|
+
);
|
|
403
|
+
client._h2conns.add(h2);
|
|
404
|
+
client._h2.set(key, h2);
|
|
405
|
+
return h2;
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
/**
|
|
409
|
+
* Does this failure prove the peer never saw the request? Only then may it be re-sent.
|
|
410
|
+
*
|
|
411
|
+
* The single provable case is a connection that ended having produced no response byte at all,
|
|
412
|
+
* which is exactly what an idle keep-alive socket the peer had already closed looks like. It
|
|
413
|
+
* surfaces two ways depending on how the peer hung up: a clean close_notify drains the record
|
|
414
|
+
* layer and the head reader hits EOF with nothing buffered, while a bare TCP close makes the
|
|
415
|
+
* record layer refuse the truncation itself. Both report `got: 0`.
|
|
416
|
+
*
|
|
417
|
+
* Everything else must fall through and be reported, a timeout above all: a request that timed out
|
|
418
|
+
* may be executing on the server at this moment, so re-sending it would apply a non-idempotent
|
|
419
|
+
* operation twice — precisely the ambiguous state this package refuses to continue in. Partial
|
|
420
|
+
* bytes disqualify too, because a peer that had begun answering had the request.
|
|
421
|
+
*
|
|
422
|
+
* @param {unknown} err
|
|
423
|
+
* @returns {boolean}
|
|
424
|
+
*/
|
|
425
|
+
function serverNeverSawIt(err) {
|
|
426
|
+
if (err instanceof UnexpectedEofError) return err.detail?.got === 0;
|
|
427
|
+
return err instanceof TunnelFetchError && err.code === codes.TLS_TRUNCATED && err.detail?.got === 0;
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
async function sendAndReceive(client, conn, current, { key, deadlines, reused }) {
|
|
431
|
+
const o = client.options;
|
|
432
|
+
const target = targetFromUrl(current.url);
|
|
433
|
+
const headers = buildHeaders(client, current, target);
|
|
434
|
+
|
|
435
|
+
const head = serializeRequestHead({
|
|
436
|
+
method: current.method,
|
|
437
|
+
target: requestTarget(current.url),
|
|
438
|
+
headers,
|
|
439
|
+
});
|
|
440
|
+
|
|
441
|
+
const writer = new ByteWriter(conn.writable);
|
|
442
|
+
try {
|
|
443
|
+
await writer.write(current.body ? concat([head, current.body]) : head);
|
|
444
|
+
} finally {
|
|
445
|
+
writer.releaseLock();
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
deadlines.beginPhase('headers');
|
|
449
|
+
const reader = conn._reader ?? new ByteReader(conn.readable);
|
|
450
|
+
conn._reader = reader; // one reader per connection, so a reused socket keeps its buffer
|
|
451
|
+
|
|
452
|
+
let headInfo;
|
|
453
|
+
try {
|
|
454
|
+
headInfo = await deadlines.race(
|
|
455
|
+
readResponseHead(reader, { maxHeaderBytes: o.limits?.maxHeaderBytes }),
|
|
456
|
+
);
|
|
457
|
+
} catch (err) {
|
|
458
|
+
// A reused connection the server had already reaped looks exactly like a truncated response,
|
|
459
|
+
// and re-sending on a fresh connection is the standard remedy. It is only safe, though, when
|
|
460
|
+
// the failure proves the server never saw the request — see serverNeverSawIt().
|
|
461
|
+
if (reused && serverNeverSawIt(err)) {
|
|
462
|
+
await client.pool.discard(conn);
|
|
463
|
+
deadlines.endPhase();
|
|
464
|
+
const fresh = await openConnection({
|
|
465
|
+
url: current.url,
|
|
466
|
+
connect: o.connect,
|
|
467
|
+
proxy: parseProxy(o.proxy ?? null),
|
|
468
|
+
trust: o.trust ?? { mode: 'system' },
|
|
469
|
+
tls: o.tls ?? {},
|
|
470
|
+
resumption: resumptionFor(client, key),
|
|
471
|
+
deps: o.deps,
|
|
472
|
+
deadlines,
|
|
473
|
+
limits: o.limits ?? {},
|
|
474
|
+
now: o.now,
|
|
475
|
+
});
|
|
476
|
+
return sendAndReceive(client, fresh, current, { key, deadlines, reused: false });
|
|
477
|
+
}
|
|
478
|
+
throw err;
|
|
479
|
+
}
|
|
480
|
+
deadlines.endPhase();
|
|
481
|
+
|
|
482
|
+
const framing = bodyFraming({
|
|
483
|
+
status: headInfo.status,
|
|
484
|
+
method: current.method,
|
|
485
|
+
headers: headInfo.headers,
|
|
486
|
+
});
|
|
487
|
+
|
|
488
|
+
if (client.jar && headInfo.setCookie?.length) {
|
|
489
|
+
client.jar.setFromResponse(current.url, headInfo.setCookie);
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
const raw = readResponseBody(reader, framing, { maxBytes: o.maxBodyBytes ?? Infinity });
|
|
493
|
+
|
|
494
|
+
// The connection goes back to the pool only when the body reaches the end its framing declared.
|
|
495
|
+
// `completed` resolving false means the caller cancelled and the stream position is unknown.
|
|
496
|
+
raw.completed.then(
|
|
497
|
+
(ok) => {
|
|
498
|
+
deadlines.dispose();
|
|
499
|
+
client.pool.release(key, conn, ok && framing.keepAliveEligible && wantsKeepAlive(headInfo));
|
|
500
|
+
},
|
|
501
|
+
() => {
|
|
502
|
+
deadlines.dispose();
|
|
503
|
+
void client.pool.discard(conn);
|
|
504
|
+
},
|
|
505
|
+
);
|
|
506
|
+
|
|
507
|
+
// The idle deadline wraps the RAW body, before any content decoding. Wrapping the decoded
|
|
508
|
+
// stream instead would judge liveness by decompressed output, and a decompressor legitimately
|
|
509
|
+
// consumes input for a while before producing any — so a healthy stream would look stalled.
|
|
510
|
+
// What "still alive" means is bytes arriving from the peer, which is exactly this stream.
|
|
511
|
+
deadlines.beginIdle();
|
|
512
|
+
const guarded = framing.kind === 'none' ? raw : withIdleDeadline(raw, deadlines);
|
|
513
|
+
const decoded = decodeResponseBody(guarded, headInfo.headers, o);
|
|
514
|
+
return buildResponse(headInfo, decoded, framing, conn);
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
// ------------------------------------------------------------------ one request/response over h2
|
|
518
|
+
|
|
519
|
+
/**
|
|
520
|
+
* The HTTP/2 counterpart of sendAndReceive. The connection is shared, so nothing here checks it
|
|
521
|
+
* out or returns it — the stream id keeps this response's bytes separate from every other stream's.
|
|
522
|
+
* The two load-bearing invariants from the h1 path are preserved deliberately: the idle deadline
|
|
523
|
+
* wraps the RAW body before any content decoding, and the completion of the body disposes the
|
|
524
|
+
* per-request deadline (it just never releases a connection, because the connection is not ours to
|
|
525
|
+
* release).
|
|
526
|
+
*/
|
|
527
|
+
async function sendAndReceiveH2(client, h2, current, { deadlines }) {
|
|
528
|
+
const o = client.options;
|
|
529
|
+
const target = targetFromUrl(current.url);
|
|
530
|
+
const { authority, headers } = buildH2Request(client, current, target);
|
|
531
|
+
|
|
532
|
+
deadlines.beginPhase('headers');
|
|
533
|
+
let head;
|
|
534
|
+
try {
|
|
535
|
+
head = await deadlines.race(
|
|
536
|
+
h2.request({
|
|
537
|
+
method: current.method,
|
|
538
|
+
scheme: current.url.protocol === 'https:' ? 'https' : 'http',
|
|
539
|
+
authority,
|
|
540
|
+
path: requestTarget(current.url) || '/',
|
|
541
|
+
headers,
|
|
542
|
+
body: current.body ?? null,
|
|
543
|
+
// The deadline's signal reaches into the connection so a headers/idle timeout resets
|
|
544
|
+
// exactly this one stream (RST_STREAM), never the shared connection or its other streams.
|
|
545
|
+
signal: deadlines.signal,
|
|
546
|
+
}),
|
|
547
|
+
);
|
|
548
|
+
} finally {
|
|
549
|
+
deadlines.endPhase();
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
if (client.jar && head.setCookie?.length) {
|
|
553
|
+
client.jar.setFromResponse(current.url, head.setCookie);
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
const raw = head.body;
|
|
557
|
+
// Dispose the per-request deadline once the body is done, however it ends. Unlike h1 there is no
|
|
558
|
+
// pool.release: the connection stays shared and alive for the next stream.
|
|
559
|
+
raw.completed.then(
|
|
560
|
+
() => deadlines.dispose(),
|
|
561
|
+
() => deadlines.dispose(),
|
|
562
|
+
);
|
|
563
|
+
|
|
564
|
+
// Same invariant as h1: the idle deadline wraps the RAW body, before content decoding, so
|
|
565
|
+
// liveness is judged by bytes arriving from the peer rather than by decompressed output.
|
|
566
|
+
deadlines.beginIdle();
|
|
567
|
+
const guarded = withIdleDeadline(raw, deadlines);
|
|
568
|
+
const decoded = decodeResponseBody(guarded, head.headers, o);
|
|
569
|
+
const framing = { kind: 'h2', keepAliveEligible: false };
|
|
570
|
+
return buildResponse(head, decoded, framing, h2);
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
/**
|
|
574
|
+
* Build the HTTP/2 request: the :authority value, and the ordered regular header fields with the
|
|
575
|
+
* pseudo-headers and every connection-specific field removed (RFC 9113 s8.2.2). Framing is the
|
|
576
|
+
* client's to declare here just as in h1 — Transfer-Encoding has no meaning in h2 and is dropped,
|
|
577
|
+
* and Content-Length is set for a body so it matches what curl sends.
|
|
578
|
+
*/
|
|
579
|
+
function buildH2Request(client, current, target) {
|
|
580
|
+
const o = client.options;
|
|
581
|
+
const headers = new Headers(current.headers);
|
|
582
|
+
const defaultPort = current.url.protocol === 'https:' ? 443 : 80;
|
|
583
|
+
const authority =
|
|
584
|
+
target.port === defaultPort ? current.url.hostname : `${current.url.hostname}:${target.port}`;
|
|
585
|
+
|
|
586
|
+
if (!headers.has('accept')) headers.set('accept', '*/*');
|
|
587
|
+
if (!headers.has('accept-encoding') && o.decompress !== false) {
|
|
588
|
+
headers.set('accept-encoding', ACCEPT_ENCODING);
|
|
589
|
+
}
|
|
590
|
+
if (client.jar) {
|
|
591
|
+
const cookie = client.jar.headerFor(current.url);
|
|
592
|
+
if (cookie) {
|
|
593
|
+
const existing = headers.get('cookie');
|
|
594
|
+
headers.set('cookie', existing ? `${existing}; ${cookie}` : cookie);
|
|
595
|
+
}
|
|
596
|
+
}
|
|
597
|
+
// Connection-specific header fields are forbidden in HTTP/2 and are the sender's to strip.
|
|
598
|
+
for (const name of ['connection', 'keep-alive', 'proxy-connection', 'transfer-encoding', 'upgrade', 'host']) {
|
|
599
|
+
headers.delete(name);
|
|
600
|
+
}
|
|
601
|
+
if (current.body) headers.set('content-length', String(current.body.byteLength));
|
|
602
|
+
else if (['POST', 'PUT', 'PATCH'].includes(current.method)) headers.set('content-length', '0');
|
|
603
|
+
else headers.delete('content-length');
|
|
604
|
+
|
|
605
|
+
// Headers iterates lowercased (RFC 9113 s8.2.1 requires lowercase names on the wire) — which is
|
|
606
|
+
// also why h1 loses caller order; h2 is no different here. Pseudo-header ORDER, the part a
|
|
607
|
+
// fingerprinter reads, is fixed in http2/connection.js buildRequestFields, not here.
|
|
608
|
+
const out = [];
|
|
609
|
+
for (const [k, v] of headers) out.push([k, v]);
|
|
610
|
+
return { authority, headers: out };
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
function wantsKeepAlive(headInfo) {
|
|
614
|
+
const connection = (headInfo.headers.get('connection') ?? '').toLowerCase();
|
|
615
|
+
if (connection.split(',').some((t) => t.trim() === 'close')) return false;
|
|
616
|
+
// HTTP/1.0 defaults to close unless it explicitly asks to stay open.
|
|
617
|
+
if (headInfo.httpVersion === '1.0') {
|
|
618
|
+
return connection.split(',').some((t) => t.trim() === 'keep-alive');
|
|
619
|
+
}
|
|
620
|
+
return true;
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
function decodeResponseBody(body, headers, options) {
|
|
624
|
+
if (options.decompress === false) return body;
|
|
625
|
+
const encoding = headers.get('content-encoding');
|
|
626
|
+
if (!encoding) return body;
|
|
627
|
+
return decodeBody(body, encoding);
|
|
628
|
+
}
|
|
629
|
+
|
|
630
|
+
function buildResponse(headInfo, body, framing, conn) {
|
|
631
|
+
const headers = new Headers();
|
|
632
|
+
for (const [k, v] of headInfo.headers) {
|
|
633
|
+
if (k === 'set-cookie') continue;
|
|
634
|
+
headers.append(k, v);
|
|
635
|
+
}
|
|
636
|
+
for (const c of headInfo.setCookie ?? []) headers.append('set-cookie', c);
|
|
637
|
+
if (headers.has('content-encoding')) {
|
|
638
|
+
// The bytes handed to the caller are decoded, so a byte count describing the encoded form
|
|
639
|
+
// would be a lie. Content-Encoding stays as information about what was on the wire.
|
|
640
|
+
headers.delete('content-length');
|
|
641
|
+
}
|
|
642
|
+
|
|
643
|
+
const useNullBody = NULL_BODY_STATUS.has(headInfo.status) || framing.kind === 'none';
|
|
644
|
+
const response = new Response(useNullBody ? null : body, {
|
|
645
|
+
status: headInfo.status,
|
|
646
|
+
statusText: headInfo.statusText,
|
|
647
|
+
headers,
|
|
648
|
+
});
|
|
649
|
+
Object.defineProperty(response, 'tunnelfetch', {
|
|
650
|
+
value: Object.freeze({ ...conn.info, httpVersion: headInfo.httpVersion, framing: framing.kind }),
|
|
651
|
+
configurable: true,
|
|
652
|
+
});
|
|
653
|
+
return response;
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
function requestTarget(url) {
|
|
657
|
+
return `${url.pathname}${url.search}`;
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
function buildHeaders(client, current, target) {
|
|
661
|
+
const headers = new Headers(current.headers);
|
|
662
|
+
// Host is derived from the URL and never carried across a redirect; the default port is omitted
|
|
663
|
+
// because a server matching virtual hosts on the literal Host value expects it that way.
|
|
664
|
+
const defaultPort = current.url.protocol === 'https:' ? 443 : 80;
|
|
665
|
+
const hostValue =
|
|
666
|
+
target.port === defaultPort ? current.url.hostname : `${current.url.hostname}:${target.port}`;
|
|
667
|
+
|
|
668
|
+
if (!headers.has('accept')) headers.set('accept', '*/*');
|
|
669
|
+
if (!headers.has('accept-encoding') && client.options.decompress !== false) {
|
|
670
|
+
// Never advertise br or zstd: the runtime has no DecompressionStream for either, so the
|
|
671
|
+
// reward for asking would be a body we cannot decode.
|
|
672
|
+
headers.set('accept-encoding', ACCEPT_ENCODING);
|
|
673
|
+
}
|
|
674
|
+
if (client.jar) {
|
|
675
|
+
const cookie = client.jar.headerFor(current.url);
|
|
676
|
+
if (cookie) {
|
|
677
|
+
const existing = headers.get('cookie');
|
|
678
|
+
headers.set('cookie', existing ? `${existing}; ${cookie}` : cookie);
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
headers.set('connection', client.options.keepAlive === false ? 'close' : 'keep-alive');
|
|
682
|
+
|
|
683
|
+
// Framing is the client's to declare, never the caller's. The body is always buffered whole by
|
|
684
|
+
// performFetch, so every request this client sends is Content-Length-framed and never chunked.
|
|
685
|
+
// A caller-supplied Transfer-Encoding is therefore always a lie (we do not chunk the body), and
|
|
686
|
+
// a caller-supplied Content-Length is at best redundant and at worst a smuggling vector: emitting
|
|
687
|
+
// Content-Length AND Transfer-Encoding together is the exact ambiguity bodyFraming() refuses on
|
|
688
|
+
// the response side (RFC 9112 §6.1: a sender MUST NOT send Content-Length with Transfer-Encoding),
|
|
689
|
+
// and a Content-Length that does not match the bytes written (e.g. one left on a bodyless GET)
|
|
690
|
+
// desynchronises the very next message. Drop both, then state the one truth.
|
|
691
|
+
headers.delete('transfer-encoding');
|
|
692
|
+
if (current.body) headers.set('content-length', String(current.body.byteLength));
|
|
693
|
+
else if (['POST', 'PUT', 'PATCH'].includes(current.method)) headers.set('content-length', '0');
|
|
694
|
+
else headers.delete('content-length');
|
|
695
|
+
|
|
696
|
+
const ordered = [['host', hostValue]];
|
|
697
|
+
for (const [k, v] of headers) {
|
|
698
|
+
if (k === 'host') continue;
|
|
699
|
+
ordered.push([k, v]);
|
|
700
|
+
}
|
|
701
|
+
return ordered;
|
|
702
|
+
}
|
|
703
|
+
|
|
704
|
+
export { CookieJar, ConnectionPool, utf8 };
|