tunnelfetch 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +28 -0
- package/README.md +617 -0
- package/README.zh-CN.md +470 -0
- package/package.json +74 -0
- package/src/client/cookies.js +429 -0
- package/src/client/decode.js +346 -0
- package/src/client/redirect.js +249 -0
- package/src/client.js +704 -0
- package/src/errors.js +181 -0
- package/src/http1/chunked.js +289 -0
- package/src/http1/index.js +10 -0
- package/src/http1/request.js +143 -0
- package/src/http1/response.js +493 -0
- package/src/http2/connection.js +1170 -0
- package/src/http2/constants.js +129 -0
- package/src/http2/frames.js +291 -0
- package/src/http2/hpack.js +420 -0
- package/src/http2/huffman.js +203 -0
- package/src/http2/index.js +21 -0
- package/src/index.js +46 -0
- package/src/pool.js +256 -0
- package/src/proxy/direct.js +62 -0
- package/src/proxy/http-connect.js +206 -0
- package/src/proxy/index.js +197 -0
- package/src/proxy/socks5.js +344 -0
- package/src/tls/aead.js +263 -0
- package/src/tls/connect.js +407 -0
- package/src/tls/constants.js +334 -0
- package/src/tls/extensions.js +376 -0
- package/src/tls/handshake-messages.js +901 -0
- package/src/tls/handshake.js +568 -0
- package/src/tls/handshake12.js +507 -0
- package/src/tls/index.js +44 -0
- package/src/tls/keyschedule.js +473 -0
- package/src/tls/record.js +872 -0
- package/src/tls/tickets.js +145 -0
- package/src/tls/transcript.js +101 -0
- package/src/tls/wire.js +224 -0
- package/src/transport.js +296 -0
- package/src/trust/der.js +551 -0
- package/src/trust/index.js +375 -0
- package/src/trust/name.js +235 -0
- package/src/trust/ocsp.js +759 -0
- package/src/trust/path.js +595 -0
- package/src/trust/roots.js +454 -0
- package/src/trust/x509.js +902 -0
- package/src/util/bytes.js +470 -0
- package/src/util/deadline.js +266 -0
- package/src/warmup-fixture.js +85 -0
- package/src/warmup.js +243 -0
- package/types/client/cookies.d.ts +159 -0
- package/types/client/decode.d.ts +54 -0
- package/types/client/redirect.d.ts +96 -0
- package/types/client.d.ts +323 -0
- package/types/errors.d.ts +141 -0
- package/types/http1/chunked.d.ts +48 -0
- package/types/http1/index.d.ts +3 -0
- package/types/http1/request.d.ts +44 -0
- package/types/http1/response.d.ts +183 -0
- package/types/http2/connection.d.ts +282 -0
- package/types/http2/constants.d.ts +95 -0
- package/types/http2/frames.d.ts +116 -0
- package/types/http2/hpack.d.ts +99 -0
- package/types/http2/huffman.d.ts +21 -0
- package/types/http2/index.d.ts +5 -0
- package/types/index.d.ts +17 -0
- package/types/pool.d.ts +135 -0
- package/types/proxy/direct.d.ts +26 -0
- package/types/proxy/http-connect.d.ts +37 -0
- package/types/proxy/index.d.ts +62 -0
- package/types/proxy/socks5.d.ts +47 -0
- package/types/tls/aead.d.ts +67 -0
- package/types/tls/connect.d.ts +280 -0
- package/types/tls/constants.d.ts +275 -0
- package/types/tls/extensions.d.ts +195 -0
- package/types/tls/handshake-messages.d.ts +430 -0
- package/types/tls/handshake.d.ts +90 -0
- package/types/tls/handshake12.d.ts +35 -0
- package/types/tls/index.d.ts +9 -0
- package/types/tls/keyschedule.d.ts +272 -0
- package/types/tls/record.d.ts +361 -0
- package/types/tls/tickets.d.ts +66 -0
- package/types/tls/transcript.d.ts +52 -0
- package/types/tls/wire.d.ts +106 -0
- package/types/transport.d.ts +222 -0
- package/types/trust/der.d.ts +239 -0
- package/types/trust/index.d.ts +194 -0
- package/types/trust/name.d.ts +33 -0
- package/types/trust/ocsp.d.ts +138 -0
- package/types/trust/path.d.ts +139 -0
- package/types/trust/roots.d.ts +36 -0
- package/types/trust/x509.d.ts +401 -0
- package/types/util/bytes.d.ts +183 -0
- package/types/util/deadline.d.ts +133 -0
- package/types/warmup-fixture.d.ts +11 -0
- package/types/warmup.d.ts +45 -0
|
@@ -0,0 +1,346 @@
|
|
|
1
|
+
// Content-Encoding and charset: the last two transforms between wire bytes and what the
|
|
2
|
+
// caller reads. Both are riddled with server-side sloppiness, so both sides are pinned here:
|
|
3
|
+
// the request side (what we advertise) and the response side (what we decode) live in one
|
|
4
|
+
// file so they cannot drift apart.
|
|
5
|
+
|
|
6
|
+
import { HttpError, codes } from '../errors.js';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* The exact Accept-Encoding value the request layer must send. The target runtime's
|
|
10
|
+
* DecompressionStream supports ONLY gzip / deflate / deflate-raw (verified empirically);
|
|
11
|
+
* advertising `br` or `zstd` would invite bytes we can never decode, turning every response
|
|
12
|
+
* from a brotli-preferring CDN into garbage. Keep this list and decodeBody in lockstep.
|
|
13
|
+
*/
|
|
14
|
+
export const ACCEPT_ENCODING = 'gzip, deflate';
|
|
15
|
+
|
|
16
|
+
/** Codings that exist and are real but that this runtime cannot decompress. */
|
|
17
|
+
const KNOWN_UNSUPPORTED = new Set(['br', 'zstd', 'compress', 'x-compress']);
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* How much decompressed output one BYOB read may deliver. A BYOB read resolves as soon as at
|
|
21
|
+
* least one byte is available — it never waits for the view to fill — so a large view cannot
|
|
22
|
+
* add latency; it only lets a fast decompressor hand over more per boundary crossing.
|
|
23
|
+
*/
|
|
24
|
+
const DECOMPRESS_READ_BYTES = 65536;
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* One decompression stage. `sniffDeflate` handles the deflate ambiguity:
|
|
28
|
+
*
|
|
29
|
+
* RFC 9110 says Content-Encoding: deflate means ZLIB-WRAPPED deflate (RFC 1950), but a large
|
|
30
|
+
* population of servers — old IIS most famously — send RAW deflate (RFC 1951) under the same
|
|
31
|
+
* name. Every interoperable client sniffs. The test: a zlib stream starts with a 2-byte header
|
|
32
|
+
* where (b0 & 0x0f) === 8 (CM = deflate) and ((b0 << 8) | b1) % 31 === 0 (the FCHECK checksum,
|
|
33
|
+
* designed for exactly this kind of validation). Raw deflate data cannot systematically fake
|
|
34
|
+
* both, so the check is reliable in practice.
|
|
35
|
+
*
|
|
36
|
+
* The output side is pull-driven and drains the decompressor with a BYOB reader when the
|
|
37
|
+
* runtime supports one (a large view per read), falling back to a default reader elsewhere.
|
|
38
|
+
* This shape is measured, not aesthetic: the target runtime's DecompressionStream emits
|
|
39
|
+
* 4096-byte chunks, and the previous wiring (pipeTo → WritableStream → TransformStream)
|
|
40
|
+
* crossed the JS/runtime boundary several times per chunk — measured on the edge at
|
|
41
|
+
* ~28 ms of CPU per MB of decompressed output for this stage alone, against ~2 ms/MB for
|
|
42
|
+
* the inflate itself. Draining with one 64 KiB read per crossing brings the stage to
|
|
43
|
+
* ~6 ms/MB (A/B-ed old-vs-new inside one isolate: 110 ms vs 23 ms for a 4 MB body).
|
|
44
|
+
* A BYOB read resolves with a partial fill the moment any output exists — verified on the
|
|
45
|
+
* edge with a stalled input, 58 KB arrived into a 1 MB view — so streaming latency is
|
|
46
|
+
* unchanged. Input is still pumped by an independent task: a decompressor legitimately
|
|
47
|
+
* consumes many input chunks before producing output, so tying input progress to output
|
|
48
|
+
* pulls would deadlock.
|
|
49
|
+
*/
|
|
50
|
+
function decompressionStage(source, coding) {
|
|
51
|
+
const srcReader = source.getReader();
|
|
52
|
+
/** Rejections here surface through the output stream; pre-observed like chunked.js does. */
|
|
53
|
+
let pumpDone = null;
|
|
54
|
+
// The returned stream must exist synchronously, but the format is only known after the
|
|
55
|
+
// 2-byte sniff. `ready` settles with the DecompressionStream once the sniff has run, or
|
|
56
|
+
// with null for a zero-byte body.
|
|
57
|
+
const ready = (async () => {
|
|
58
|
+
// Buffer up to 2 bytes so the deflate sniff (and the empty-body check) can see them.
|
|
59
|
+
const head = [];
|
|
60
|
+
let headLen = 0;
|
|
61
|
+
while (headLen < 2) {
|
|
62
|
+
const { value, done } = await srcReader.read();
|
|
63
|
+
if (done) break;
|
|
64
|
+
if (value && value.byteLength) {
|
|
65
|
+
head.push(value);
|
|
66
|
+
headLen += value.byteLength;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
if (headLen === 0) {
|
|
70
|
+
// A zero-byte body under a Content-Encoding header is not a valid compressed stream,
|
|
71
|
+
// but CDNs emit it constantly (204-shaped responses, HEAD-derived bodies). Zero bytes
|
|
72
|
+
// in, zero bytes out is the only non-destructive reading, and it is what browsers do.
|
|
73
|
+
return null;
|
|
74
|
+
}
|
|
75
|
+
let format = coding;
|
|
76
|
+
if (coding === 'deflate') {
|
|
77
|
+
const b0 = headLen >= 1 ? firstBytes(head, 0) : -1;
|
|
78
|
+
const b1 = headLen >= 2 ? firstBytes(head, 1) : -1;
|
|
79
|
+
const isZlib = b1 >= 0 && (b0 & 0x0f) === 8 && ((b0 << 8) | b1) % 31 === 0;
|
|
80
|
+
format = isZlib ? 'deflate' : 'deflate-raw';
|
|
81
|
+
}
|
|
82
|
+
const ds = new DecompressionStream(format);
|
|
83
|
+
const dsWriter = ds.writable.getWriter();
|
|
84
|
+
pumpDone = (async () => {
|
|
85
|
+
try {
|
|
86
|
+
for (const chunk of head) await dsWriter.write(chunk);
|
|
87
|
+
for (;;) {
|
|
88
|
+
const { value, done } = await srcReader.read();
|
|
89
|
+
if (done) break;
|
|
90
|
+
if (value && value.byteLength) await dsWriter.write(value);
|
|
91
|
+
}
|
|
92
|
+
await dsWriter.close();
|
|
93
|
+
} catch (e) {
|
|
94
|
+
// Either the source failed (its error must reach the consumer, so error the
|
|
95
|
+
// decompressor's output with it) or the decompressor rejected its input (already
|
|
96
|
+
// errored; the abort is a no-op). Both tear the source down.
|
|
97
|
+
await dsWriter.abort(e).catch(() => {});
|
|
98
|
+
await srcReader.cancel(e).catch(() => {});
|
|
99
|
+
throw e;
|
|
100
|
+
}
|
|
101
|
+
})();
|
|
102
|
+
pumpDone.catch(() => {});
|
|
103
|
+
return ds;
|
|
104
|
+
})();
|
|
105
|
+
ready.catch(() => {});
|
|
106
|
+
|
|
107
|
+
/** @type {ReadableStreamBYOBReader | ReadableStreamDefaultReader<Uint8Array> | null} */
|
|
108
|
+
let out = null;
|
|
109
|
+
let byob = false;
|
|
110
|
+
const wrap = (e) =>
|
|
111
|
+
e instanceof HttpError
|
|
112
|
+
? e
|
|
113
|
+
: new HttpError(
|
|
114
|
+
codes.HTTP_CONTENT_ENCODING,
|
|
115
|
+
`decoding "${coding}" failed: ${e?.message ?? e}`,
|
|
116
|
+
{ coding },
|
|
117
|
+
);
|
|
118
|
+
|
|
119
|
+
return new ReadableStream({
|
|
120
|
+
async pull(c) {
|
|
121
|
+
try {
|
|
122
|
+
const ds = await ready;
|
|
123
|
+
if (ds === null) {
|
|
124
|
+
c.close();
|
|
125
|
+
return;
|
|
126
|
+
}
|
|
127
|
+
if (out === null) {
|
|
128
|
+
try {
|
|
129
|
+
out = ds.readable.getReader({ mode: 'byob' });
|
|
130
|
+
byob = true;
|
|
131
|
+
} catch {
|
|
132
|
+
// The runtime's DecompressionStream readable is not a byte stream (Node's is
|
|
133
|
+
// not); a default reader delivers the same bytes in the decompressor's own
|
|
134
|
+
// chunking.
|
|
135
|
+
out = ds.readable.getReader();
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
for (;;) {
|
|
139
|
+
const { value, done } = byob
|
|
140
|
+
? await /** @type {ReadableStreamBYOBReader} */ (out)
|
|
141
|
+
.read(new Uint8Array(DECOMPRESS_READ_BYTES))
|
|
142
|
+
: await /** @type {ReadableStreamDefaultReader<Uint8Array>} */ (out).read();
|
|
143
|
+
if (done) {
|
|
144
|
+
// Clean output EOF implies the input pump already closed the decompressor
|
|
145
|
+
// successfully; awaiting it here surfaces any failure that raced the close.
|
|
146
|
+
if (pumpDone) await pumpDone;
|
|
147
|
+
c.close();
|
|
148
|
+
return;
|
|
149
|
+
}
|
|
150
|
+
if (value.byteLength === 0) continue; // legal, carries nothing; keep reading
|
|
151
|
+
c.enqueue(value);
|
|
152
|
+
return;
|
|
153
|
+
}
|
|
154
|
+
} catch (e) {
|
|
155
|
+
// Corrupt compressed data surfaces to the consumer as a stream error, wrapped so the
|
|
156
|
+
// caller sees which coding failed rather than a bare zlib message.
|
|
157
|
+
const err = wrap(e);
|
|
158
|
+
await srcReader.cancel(err).catch(() => {});
|
|
159
|
+
c.error(err);
|
|
160
|
+
}
|
|
161
|
+
},
|
|
162
|
+
async cancel(reason) {
|
|
163
|
+
// The consumer abandoned the decoded body: cancel the source and release the
|
|
164
|
+
// decompressor, exactly as the failure path does — an abandoned stream must still tear
|
|
165
|
+
// the connection down (the pool only reuses a connection whose body reached its framed
|
|
166
|
+
// end). The source goes FIRST: the sniff may still be awaiting it, and `ready` cannot
|
|
167
|
+
// settle until that read resolves.
|
|
168
|
+
await srcReader.cancel(reason).catch(() => {});
|
|
169
|
+
try {
|
|
170
|
+
if (out) {
|
|
171
|
+
await out.cancel(reason);
|
|
172
|
+
} else {
|
|
173
|
+
const ds = await ready.catch(() => null);
|
|
174
|
+
if (ds) await ds.readable.cancel(reason);
|
|
175
|
+
}
|
|
176
|
+
} catch {
|
|
177
|
+
/* the decompressor may already be errored */
|
|
178
|
+
}
|
|
179
|
+
},
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/** Byte at logical offset `i` across the buffered head chunks. */
|
|
184
|
+
function firstBytes(chunks, i) {
|
|
185
|
+
for (const c of chunks) {
|
|
186
|
+
if (i < c.byteLength) return c[i];
|
|
187
|
+
i -= c.byteLength;
|
|
188
|
+
}
|
|
189
|
+
return -1;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/**
|
|
193
|
+
* Undo the response's Content-Encoding.
|
|
194
|
+
*
|
|
195
|
+
* @param {ReadableStream<Uint8Array>} stream the raw body
|
|
196
|
+
* @param {string|null|undefined} contentEncoding the Content-Encoding header value; a
|
|
197
|
+
* comma-separated list names codings in the order the SERVER applied them, so decoding
|
|
198
|
+
* applies them in reverse.
|
|
199
|
+
* @returns {ReadableStream<Uint8Array>} decoded bytes
|
|
200
|
+
*/
|
|
201
|
+
export function decodeBody(stream, contentEncoding) {
|
|
202
|
+
const codings = (contentEncoding ?? '')
|
|
203
|
+
.split(',')
|
|
204
|
+
.map((c) => c.trim().toLowerCase())
|
|
205
|
+
.filter((c) => c !== '');
|
|
206
|
+
// Validate the whole list before wiring any stage, so an unsupported coding is a clean
|
|
207
|
+
// synchronous throw rather than a half-consumed stream.
|
|
208
|
+
for (const coding of codings) {
|
|
209
|
+
if (coding === 'gzip' || coding === 'x-gzip' || coding === 'deflate' || coding === 'identity') {
|
|
210
|
+
continue;
|
|
211
|
+
}
|
|
212
|
+
if (KNOWN_UNSUPPORTED.has(coding)) {
|
|
213
|
+
throw new HttpError(
|
|
214
|
+
codes.HTTP_CONTENT_ENCODING,
|
|
215
|
+
`Content-Encoding "${coding}" is not decodable: this runtime's DecompressionStream ` +
|
|
216
|
+
'supports only gzip/deflate/deflate-raw, which is why this client never advertises ' +
|
|
217
|
+
`"${coding}" in Accept-Encoding — the server should not have sent it`,
|
|
218
|
+
{ coding },
|
|
219
|
+
);
|
|
220
|
+
}
|
|
221
|
+
throw new HttpError(codes.HTTP_CONTENT_ENCODING, `unknown Content-Encoding "${coding}"`, {
|
|
222
|
+
coding,
|
|
223
|
+
});
|
|
224
|
+
}
|
|
225
|
+
let out = stream;
|
|
226
|
+
for (let i = codings.length - 1; i >= 0; i--) {
|
|
227
|
+
const coding = codings[i];
|
|
228
|
+
if (coding === 'identity') continue; // no-op by definition
|
|
229
|
+
out = decompressionStage(out, coding === 'x-gzip' ? 'gzip' : coding);
|
|
230
|
+
}
|
|
231
|
+
return out;
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
// --------------------------------------------------------------------------- charset
|
|
235
|
+
|
|
236
|
+
/**
|
|
237
|
+
* Extract the charset parameter from a Content-Type value, handling quoting and other
|
|
238
|
+
* parameters: `text/html; boundary=x; charset="ISO-8859-4"` -> 'iso-8859-4'.
|
|
239
|
+
* @param {string | null | undefined} contentType
|
|
240
|
+
* @returns {string|null} lowercased charset label, or null when none is declared
|
|
241
|
+
*/
|
|
242
|
+
export function charsetFromContentType(contentType) {
|
|
243
|
+
if (!contentType) return null;
|
|
244
|
+
// Parameters after the first ';'. A quoted-string value may contain ';', which a naive
|
|
245
|
+
// split would sever — scan parameters with a regex that consumes quoted strings whole.
|
|
246
|
+
const re = /;\s*([^=;\s]+)\s*=\s*("(?:[^"\\]|\\.)*"|[^;]*)/g;
|
|
247
|
+
let m;
|
|
248
|
+
let charset = null;
|
|
249
|
+
while ((m = re.exec(contentType)) !== null) {
|
|
250
|
+
if (m[1].toLowerCase() !== 'charset') continue;
|
|
251
|
+
let v = m[2].trim();
|
|
252
|
+
if (v.length >= 2 && v[0] === '"' && v.at(-1) === '"') {
|
|
253
|
+
v = v.slice(1, -1).replace(/\\(.)/g, '$1');
|
|
254
|
+
}
|
|
255
|
+
if (v !== '') charset = v.toLowerCase(); // last occurrence wins, like header params do
|
|
256
|
+
}
|
|
257
|
+
return charset;
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
/** BOM sniff: the byte-order mark is ground truth about the bytes that follow. */
|
|
261
|
+
function bomCharset(bytes) {
|
|
262
|
+
if (!bytes || bytes.byteLength < 2) return null;
|
|
263
|
+
if (bytes.byteLength >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) {
|
|
264
|
+
return 'utf-8';
|
|
265
|
+
}
|
|
266
|
+
if (bytes[0] === 0xfe && bytes[1] === 0xff) return 'utf-16be';
|
|
267
|
+
if (bytes[0] === 0xff && bytes[1] === 0xfe) return 'utf-16le';
|
|
268
|
+
return null;
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
/** Case-insensitive `<meta charset>` prescan over the first 1024 bytes of an HTML body. */
|
|
272
|
+
function metaCharset(bytes) {
|
|
273
|
+
const window = bytes.subarray(0, 1024);
|
|
274
|
+
// The prescan is defined over ASCII bytes; decoding as latin-1 maps every byte 1:1 to a
|
|
275
|
+
// code unit, so the regex indexes cannot be shifted by multi-byte sequences.
|
|
276
|
+
let s = '';
|
|
277
|
+
for (let i = 0; i < window.byteLength; i++) s += String.fromCharCode(window[i]);
|
|
278
|
+
s = s.toLowerCase();
|
|
279
|
+
// Both meta forms declare with `charset=`: <meta charset=utf-8> directly, and
|
|
280
|
+
// <meta http-equiv=content-type content="text/html; charset=utf-8"> inside content=.
|
|
281
|
+
const tagRe = /<meta\b[^>]*>/g;
|
|
282
|
+
let tag;
|
|
283
|
+
while ((tag = tagRe.exec(s)) !== null) {
|
|
284
|
+
const m = /charset\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s;"'>]+))/.exec(tag[0]);
|
|
285
|
+
if (m) {
|
|
286
|
+
const label = (m[1] ?? m[2] ?? m[3] ?? '').trim();
|
|
287
|
+
if (label) return label;
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
return null;
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
/**
|
|
294
|
+
* Decide the charset for a response body.
|
|
295
|
+
*
|
|
296
|
+
* Precedence: BOM > Content-Type charset parameter > (text/html only) meta prescan > utf-8.
|
|
297
|
+
* The BOM outranks even an explicit header because it describes the actual bytes, and servers
|
|
298
|
+
* that recode content routinely forget to update the header; this is WHATWG "decode" order.
|
|
299
|
+
* The utf-8 default matches Response.text() in fetch — for a client whose callers are code,
|
|
300
|
+
* matching fetch is worth more than matching the legacy HTML default of windows-1252.
|
|
301
|
+
*
|
|
302
|
+
* @param {string|null|undefined} contentType
|
|
303
|
+
* @param {Uint8Array} [bodyPrefix] the first bytes of the body (>= 1024 to satisfy the prescan)
|
|
304
|
+
* @returns {string} a charset label for decodeText
|
|
305
|
+
*/
|
|
306
|
+
export function charsetFor(contentType, bodyPrefix) {
|
|
307
|
+
const bom = bodyPrefix ? bomCharset(bodyPrefix) : null;
|
|
308
|
+
if (bom) return bom;
|
|
309
|
+
const declared = charsetFromContentType(contentType);
|
|
310
|
+
if (declared) return declared;
|
|
311
|
+
const mime = (contentType ?? '').split(';')[0].trim().toLowerCase();
|
|
312
|
+
if (mime === 'text/html' && bodyPrefix) {
|
|
313
|
+
const meta = metaCharset(bodyPrefix);
|
|
314
|
+
if (meta) return meta.toLowerCase();
|
|
315
|
+
}
|
|
316
|
+
return 'utf-8';
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
/**
|
|
320
|
+
* Decode bytes with a charset label.
|
|
321
|
+
*
|
|
322
|
+
* TextDecoder implements the WHATWG encoding registry, which is the alias table every browser
|
|
323
|
+
* uses. Note one alias that looks like a bug and is not: `iso-8859-1` (and `latin1`, `ascii`)
|
|
324
|
+
* maps to windows-1252, per WHATWG — the bytes 0x80-0x9F decode to the punctuation everyone
|
|
325
|
+
* actually means, not C1 controls. A BOM matching the charset is stripped (TextDecoder default),
|
|
326
|
+
* which is also what Response.text() does.
|
|
327
|
+
*
|
|
328
|
+
* Throws HttpError (HTTP_CHARSET) for a label outside the WHATWG encoding registry.
|
|
329
|
+
*
|
|
330
|
+
* @param {Uint8Array} bytes
|
|
331
|
+
* @param {string} [charset]
|
|
332
|
+
* @returns {string}
|
|
333
|
+
*/
|
|
334
|
+
export function decodeText(bytes, charset = 'utf-8') {
|
|
335
|
+
let decoder;
|
|
336
|
+
try {
|
|
337
|
+
decoder = new TextDecoder(charset);
|
|
338
|
+
} catch {
|
|
339
|
+
throw new HttpError(
|
|
340
|
+
codes.HTTP_CHARSET,
|
|
341
|
+
`charset label "${charset}" is not a supported encoding`,
|
|
342
|
+
{ charset },
|
|
343
|
+
);
|
|
344
|
+
}
|
|
345
|
+
return decoder.decode(bytes);
|
|
346
|
+
}
|
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
// Redirect policy: which statuses redirect, how the method and body survive the hop, and —
|
|
2
|
+
// the part that actually matters — which headers must NOT survive it.
|
|
3
|
+
//
|
|
4
|
+
// The security invariant: `Authorization`, `Cookie` and `Proxy-Authorization` never cross an
|
|
5
|
+
// origin boundary. A server you trusted enough to send credentials to can redirect you anywhere,
|
|
6
|
+
// and forwarding the header to that "anywhere" hands your bearer token to a third party. Every
|
|
7
|
+
// mainstream client has shipped this bug at least once (CVE-2018-1000007 in curl, CVE-2022-0155
|
|
8
|
+
// in follow-redirects); the stripping logic below is written to be auditable line by line.
|
|
9
|
+
|
|
10
|
+
import { HttpError, LimitError, ConfigError, codes } from '../errors.js';
|
|
11
|
+
|
|
12
|
+
/** Statuses that trigger a redirect for our purposes. 300 has no defined Location semantics,
|
|
13
|
+
* 304 is a cache signal, 305/306 are deprecated/reserved — none of them redirect. */
|
|
14
|
+
const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
|
|
15
|
+
|
|
16
|
+
export const DEFAULT_MAX_REDIRECTS = 20;
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* @param {number} status
|
|
20
|
+
* @param {string} [method] accepted for API symmetry; the status alone decides, because even a
|
|
21
|
+
* combination we will rewrite (303 + POST) is still a redirect — it just mutates the method.
|
|
22
|
+
* @returns {boolean}
|
|
23
|
+
*/
|
|
24
|
+
export function shouldRedirect(status, method) { // eslint-disable-line no-unused-vars
|
|
25
|
+
return REDIRECT_STATUSES.has(status);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Method rewriting per RFC 9110 s15.4, tempered by reality:
|
|
30
|
+
*
|
|
31
|
+
* - 301/302: the RFC says the method SHOULD be preserved, but every browser and every mainstream
|
|
32
|
+
* client (curl, undici, Go net/http) rewrites POST -> GET and drops the body, and has for
|
|
33
|
+
* twenty years. Servers are written against that behaviour, so preserving POST here would be
|
|
34
|
+
* "correct" and non-interoperable. We match the herd. Methods other than POST are preserved.
|
|
35
|
+
* - 303: "see other" — ANY method except HEAD becomes GET and the body is dropped. HEAD stays
|
|
36
|
+
* HEAD because a GET would fetch a body the caller never asked for.
|
|
37
|
+
* - 307/308: method and body are preserved by definition; that is the whole point of these codes.
|
|
38
|
+
*/
|
|
39
|
+
function rewriteMethod(status, method) {
|
|
40
|
+
const m = method.toUpperCase();
|
|
41
|
+
if (status === 303) return m === 'HEAD' ? 'HEAD' : 'GET';
|
|
42
|
+
if ((status === 301 || status === 302) && m === 'POST') return 'GET';
|
|
43
|
+
return m;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** Default ports so that http://a and http://a:80 compare as the same origin. */
|
|
47
|
+
const DEFAULT_PORT = { 'http:': '80', 'https:': '443' };
|
|
48
|
+
const effectivePort = (url) => url.port || DEFAULT_PORT[url.protocol] || '';
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Percent-encode bytes above 0x7F in a Location value.
|
|
52
|
+
*
|
|
53
|
+
* Header values are opaque octets which our HTTP layer decodes as latin-1, so a code unit in
|
|
54
|
+
* 0x80..0xFF stands for exactly one raw wire byte. We percent-encode that byte AS ITSELF rather
|
|
55
|
+
* than letting the URL parser re-encode it as UTF-8, because a server that sent byte 0xE9 meant
|
|
56
|
+
* byte 0xE9 (typically a latin-1 filename); %C3%A9 would name a different resource. Code units
|
|
57
|
+
* above 0xFF cannot have come off the wire at all, so they are rejected — fail closed rather
|
|
58
|
+
* than guess an encoding for a value we were never sent.
|
|
59
|
+
*/
|
|
60
|
+
function encodeLocationOctets(location) {
|
|
61
|
+
let out = '';
|
|
62
|
+
for (let i = 0; i < location.length; i++) {
|
|
63
|
+
const cu = location.charCodeAt(i);
|
|
64
|
+
if (cu <= 0x7f) out += location[i];
|
|
65
|
+
else if (cu <= 0xff) out += '%' + cu.toString(16).toUpperCase().padStart(2, '0');
|
|
66
|
+
else {
|
|
67
|
+
throw new HttpError(
|
|
68
|
+
codes.REDIRECT_INVALID_LOCATION,
|
|
69
|
+
`Location contains code unit U+${cu.toString(16).toUpperCase().padStart(4, '0')}, ` +
|
|
70
|
+
'which cannot appear in a header value decoded from the wire',
|
|
71
|
+
{ location },
|
|
72
|
+
);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
return out;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Resolve a Location header against the current URL, enforcing the scheme allow-list.
|
|
80
|
+
* Throws HttpError for a missing/unparseable Location or a non-http(s) scheme.
|
|
81
|
+
* @param {URL} currentUrl
|
|
82
|
+
* @param {string|null|undefined} location
|
|
83
|
+
* @returns {URL}
|
|
84
|
+
*/
|
|
85
|
+
export function resolveLocation(currentUrl, location) {
|
|
86
|
+
if (location === null || location === undefined || location.trim() === '') {
|
|
87
|
+
throw new HttpError(
|
|
88
|
+
codes.REDIRECT_INVALID_LOCATION,
|
|
89
|
+
'redirect response has no usable Location header',
|
|
90
|
+
{ location: location ?? null },
|
|
91
|
+
);
|
|
92
|
+
}
|
|
93
|
+
const trimmed = location.trim();
|
|
94
|
+
const encoded = encodeLocationOctets(trimmed);
|
|
95
|
+
let next;
|
|
96
|
+
try {
|
|
97
|
+
next = new URL(encoded, currentUrl);
|
|
98
|
+
} catch {
|
|
99
|
+
throw new HttpError(codes.REDIRECT_INVALID_LOCATION, `Location "${trimmed}" does not parse`, {
|
|
100
|
+
location: trimmed,
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
// Allow-list, not block-list: file:, data:, javascript:, blob:, ftp: and anything invented
|
|
104
|
+
// later are all equally unable to be fetched by this client and equally useful to an attacker.
|
|
105
|
+
if (next.protocol !== 'http:' && next.protocol !== 'https:') {
|
|
106
|
+
throw new HttpError(
|
|
107
|
+
codes.REDIRECT_SCHEME,
|
|
108
|
+
`redirect to unsupported scheme "${next.protocol.slice(0, -1)}" refused (${trimmed})`,
|
|
109
|
+
{ scheme: next.protocol.slice(0, -1), location: trimmed },
|
|
110
|
+
);
|
|
111
|
+
}
|
|
112
|
+
// WHATWG fetch: if the Location has no fragment, the original URL's fragment is carried over.
|
|
113
|
+
// "No fragment" means no '#' in the raw value — a trailing bare '#' is an explicit empty
|
|
114
|
+
// fragment and must NOT inherit. URL resolution never inherits the base fragment on its own.
|
|
115
|
+
if (!trimmed.includes('#') && currentUrl.hash) next.hash = currentUrl.hash;
|
|
116
|
+
return next;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/** Headers that describe the body being dropped alongside a method rewrite to GET. */
|
|
120
|
+
const BODY_HEADERS = ['content-length', 'content-type', 'content-encoding', 'transfer-encoding'];
|
|
121
|
+
|
|
122
|
+
/** Credentials that must never cross an origin boundary. */
|
|
123
|
+
const CREDENTIAL_HEADERS = ['authorization', 'cookie', 'proxy-authorization'];
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* A request as the redirect engine consumes it. Bodies stay in whatever form the caller holds
|
|
127
|
+
* them; this layer only decides whether they survive the hop, never reads them.
|
|
128
|
+
* @typedef {Uint8Array | string | ReadableStream<Uint8Array> | null} RedirectBody
|
|
129
|
+
* @typedef {object} RedirectableRequest
|
|
130
|
+
* @property {string} method
|
|
131
|
+
* @property {string | URL} url
|
|
132
|
+
* @property {Headers | Record<string, string>} [headers]
|
|
133
|
+
* @property {RedirectBody} [body]
|
|
134
|
+
*/
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* The follow-up request. `url` and `headers` are always normalised instances; `body` is the
|
|
138
|
+
* caller's own value passed through, or null when the method rewrite dropped it.
|
|
139
|
+
* @typedef {object} NextRequest
|
|
140
|
+
* @property {string} method
|
|
141
|
+
* @property {URL} url
|
|
142
|
+
* @property {Headers} headers credential and body-describing headers already stripped per the
|
|
143
|
+
* rules above
|
|
144
|
+
* @property {RedirectBody} body
|
|
145
|
+
*/
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* @typedef {object} NextRequestOptions
|
|
149
|
+
* @property {number} [maxRedirects] default 20
|
|
150
|
+
* @property {string[]} [history] pass the SAME array across every hop of one logical fetch; it
|
|
151
|
+
* is both the loop detector and the hop counter
|
|
152
|
+
*/
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* Compute the follow-up request for a redirect response. Throws rather than returning a
|
|
156
|
+
* failure: LimitError past `maxRedirects`, HttpError for loops and bad Locations, ConfigError
|
|
157
|
+
* for a non-redirect status or an unreplayable stream body on 307/308.
|
|
158
|
+
*
|
|
159
|
+
* @param {RedirectableRequest} current
|
|
160
|
+
* @param {{ status: number, headers: Headers|Record<string,string> }} response
|
|
161
|
+
* @param {NextRequestOptions} [options]
|
|
162
|
+
* @returns {NextRequest}
|
|
163
|
+
*/
|
|
164
|
+
export function nextRequest(current, response, options = {}) {
|
|
165
|
+
const { maxRedirects = DEFAULT_MAX_REDIRECTS, history = [] } = options;
|
|
166
|
+
const status = response.status;
|
|
167
|
+
if (!shouldRedirect(status)) {
|
|
168
|
+
throw new ConfigError(codes.CONFIG_INVALID, `status ${status} is not a redirect`, { status });
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
// Hop budget first: `history` gains one entry per hop taken, so its length at entry is the
|
|
172
|
+
// number of redirects already followed. Checked before any parsing so a hostile chain of
|
|
173
|
+
// malformed Locations still costs at most maxRedirects rounds.
|
|
174
|
+
if (history.length >= maxRedirects) {
|
|
175
|
+
throw new LimitError(
|
|
176
|
+
codes.LIMIT_REDIRECTS,
|
|
177
|
+
`stopped after ${history.length} redirects (limit ${maxRedirects})`,
|
|
178
|
+
{ limit: maxRedirects },
|
|
179
|
+
);
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
const currentUrl = current.url instanceof URL ? current.url : new URL(current.url);
|
|
183
|
+
const currentHeaders = new Headers(current.headers ?? {});
|
|
184
|
+
const responseHeaders = new Headers(response.headers ?? {});
|
|
185
|
+
|
|
186
|
+
const nextUrl = resolveLocation(currentUrl, responseHeaders.get('location'));
|
|
187
|
+
const method = rewriteMethod(status, current.method ?? 'GET');
|
|
188
|
+
const methodRewritten = method !== (current.method ?? 'GET').toUpperCase();
|
|
189
|
+
|
|
190
|
+
// Loop detection keyed on (method, url): GET /a and POST /a are different requests, and a
|
|
191
|
+
// 303 chain legitimately revisits a URL once with the rewritten method. The fragment is not
|
|
192
|
+
// part of the key — the wire request is identical with or without it.
|
|
193
|
+
const keyOf = (m, u) => {
|
|
194
|
+
const bare = new URL(u);
|
|
195
|
+
bare.hash = '';
|
|
196
|
+
return `${m} ${bare.href}`;
|
|
197
|
+
};
|
|
198
|
+
history.push(keyOf((current.method ?? 'GET').toUpperCase(), currentUrl));
|
|
199
|
+
const nextKey = keyOf(method, nextUrl);
|
|
200
|
+
if (history.includes(nextKey)) {
|
|
201
|
+
throw new HttpError(codes.REDIRECT_LOOP, `redirect loop: already visited ${nextKey}`, {
|
|
202
|
+
key: nextKey,
|
|
203
|
+
hops: history.length,
|
|
204
|
+
});
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
// ---- header stripping: the security-critical section ----------------------------------
|
|
208
|
+
//
|
|
209
|
+
// "Same origin" is scheme AND host AND port, with default ports normalised so http://a and
|
|
210
|
+
// http://a:80 agree. The three checks are computed separately (not via URL.origin string
|
|
211
|
+
// comparison) so that each one is individually visible to an auditor, and so a change in any
|
|
212
|
+
// single component demonstrably trips the strip. This single rule already subsumes the
|
|
213
|
+
// special cases the spec calls out — a host-only change, and an https->http downgrade on the
|
|
214
|
+
// same host, both differ in at least one component and therefore strip.
|
|
215
|
+
const headers = new Headers(currentHeaders);
|
|
216
|
+
const schemeChanged = nextUrl.protocol !== currentUrl.protocol;
|
|
217
|
+
const hostChanged = nextUrl.hostname !== currentUrl.hostname;
|
|
218
|
+
const portChanged = effectivePort(nextUrl) !== effectivePort(currentUrl);
|
|
219
|
+
if (schemeChanged || hostChanged || portChanged) {
|
|
220
|
+
for (const h of CREDENTIAL_HEADERS) headers.delete(h);
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
// Host is derived from the target URL by the request serialiser; a carried-over Host would
|
|
224
|
+
// silently re-route a cross-origin hop at the HTTP layer (request smuggling adjacent).
|
|
225
|
+
headers.delete('host');
|
|
226
|
+
|
|
227
|
+
let body = current.body ?? null;
|
|
228
|
+
if (methodRewritten) {
|
|
229
|
+
// The body is gone, so every header describing it must go too, or the next server sees a
|
|
230
|
+
// Content-Length for bytes that never arrive and stalls or smuggles.
|
|
231
|
+
body = null;
|
|
232
|
+
for (const h of BODY_HEADERS) headers.delete(h);
|
|
233
|
+
} else if (status === 307 || status === 308) {
|
|
234
|
+
// 307/308 must replay the body verbatim. A ReadableStream that has already been consumed
|
|
235
|
+
// cannot be replayed, and silently sending an empty body would corrupt the request (an
|
|
236
|
+
// upload that "succeeds" with zero bytes). Distinct, detectable failure instead: callers
|
|
237
|
+
// that want 307-following with a stream body must buffer it or supply a fresh stream.
|
|
238
|
+
if (typeof ReadableStream !== 'undefined' && body instanceof ReadableStream) {
|
|
239
|
+
throw new ConfigError(
|
|
240
|
+
codes.CONFIG_INVALID,
|
|
241
|
+
`${status} redirect requires replaying the request body, but the body is a stream ` +
|
|
242
|
+
'that cannot be replayed; buffer the body to follow this redirect',
|
|
243
|
+
{ reason: 'body-not-replayable', status },
|
|
244
|
+
);
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
return { method, url: nextUrl, headers, body };
|
|
249
|
+
}
|