tunnelfetch 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +28 -0
- package/README.md +617 -0
- package/README.zh-CN.md +470 -0
- package/package.json +74 -0
- package/src/client/cookies.js +429 -0
- package/src/client/decode.js +346 -0
- package/src/client/redirect.js +249 -0
- package/src/client.js +704 -0
- package/src/errors.js +181 -0
- package/src/http1/chunked.js +289 -0
- package/src/http1/index.js +10 -0
- package/src/http1/request.js +143 -0
- package/src/http1/response.js +493 -0
- package/src/http2/connection.js +1170 -0
- package/src/http2/constants.js +129 -0
- package/src/http2/frames.js +291 -0
- package/src/http2/hpack.js +420 -0
- package/src/http2/huffman.js +203 -0
- package/src/http2/index.js +21 -0
- package/src/index.js +46 -0
- package/src/pool.js +256 -0
- package/src/proxy/direct.js +62 -0
- package/src/proxy/http-connect.js +206 -0
- package/src/proxy/index.js +197 -0
- package/src/proxy/socks5.js +344 -0
- package/src/tls/aead.js +263 -0
- package/src/tls/connect.js +407 -0
- package/src/tls/constants.js +334 -0
- package/src/tls/extensions.js +376 -0
- package/src/tls/handshake-messages.js +901 -0
- package/src/tls/handshake.js +568 -0
- package/src/tls/handshake12.js +507 -0
- package/src/tls/index.js +44 -0
- package/src/tls/keyschedule.js +473 -0
- package/src/tls/record.js +872 -0
- package/src/tls/tickets.js +145 -0
- package/src/tls/transcript.js +101 -0
- package/src/tls/wire.js +224 -0
- package/src/transport.js +296 -0
- package/src/trust/der.js +551 -0
- package/src/trust/index.js +375 -0
- package/src/trust/name.js +235 -0
- package/src/trust/ocsp.js +759 -0
- package/src/trust/path.js +595 -0
- package/src/trust/roots.js +454 -0
- package/src/trust/x509.js +902 -0
- package/src/util/bytes.js +470 -0
- package/src/util/deadline.js +266 -0
- package/src/warmup-fixture.js +85 -0
- package/src/warmup.js +243 -0
- package/types/client/cookies.d.ts +159 -0
- package/types/client/decode.d.ts +54 -0
- package/types/client/redirect.d.ts +96 -0
- package/types/client.d.ts +323 -0
- package/types/errors.d.ts +141 -0
- package/types/http1/chunked.d.ts +48 -0
- package/types/http1/index.d.ts +3 -0
- package/types/http1/request.d.ts +44 -0
- package/types/http1/response.d.ts +183 -0
- package/types/http2/connection.d.ts +282 -0
- package/types/http2/constants.d.ts +95 -0
- package/types/http2/frames.d.ts +116 -0
- package/types/http2/hpack.d.ts +99 -0
- package/types/http2/huffman.d.ts +21 -0
- package/types/http2/index.d.ts +5 -0
- package/types/index.d.ts +17 -0
- package/types/pool.d.ts +135 -0
- package/types/proxy/direct.d.ts +26 -0
- package/types/proxy/http-connect.d.ts +37 -0
- package/types/proxy/index.d.ts +62 -0
- package/types/proxy/socks5.d.ts +47 -0
- package/types/tls/aead.d.ts +67 -0
- package/types/tls/connect.d.ts +280 -0
- package/types/tls/constants.d.ts +275 -0
- package/types/tls/extensions.d.ts +195 -0
- package/types/tls/handshake-messages.d.ts +430 -0
- package/types/tls/handshake.d.ts +90 -0
- package/types/tls/handshake12.d.ts +35 -0
- package/types/tls/index.d.ts +9 -0
- package/types/tls/keyschedule.d.ts +272 -0
- package/types/tls/record.d.ts +361 -0
- package/types/tls/tickets.d.ts +66 -0
- package/types/tls/transcript.d.ts +52 -0
- package/types/tls/wire.d.ts +106 -0
- package/types/transport.d.ts +222 -0
- package/types/trust/der.d.ts +239 -0
- package/types/trust/index.d.ts +194 -0
- package/types/trust/name.d.ts +33 -0
- package/types/trust/ocsp.d.ts +138 -0
- package/types/trust/path.d.ts +139 -0
- package/types/trust/roots.d.ts +36 -0
- package/types/trust/x509.d.ts +401 -0
- package/types/util/bytes.d.ts +183 -0
- package/types/util/deadline.d.ts +133 -0
- package/types/warmup-fixture.d.ts +11 -0
- package/types/warmup.d.ts +45 -0
package/src/errors.js
ADDED
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
// Typed errors.
|
|
2
|
+
//
|
|
3
|
+
// Discipline: an error must be specific enough that one log line decides whether a missing
|
|
4
|
+
// feature is worth implementing. "unsupported cipher" is useless; "server selected 0xc02f
|
|
5
|
+
// (TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256) under TLS 1.2, not implemented" is actionable.
|
|
6
|
+
// Every throw site therefore carries the concrete value the peer asked for, in `detail`.
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* One of the stable machine-readable codes in {@link codes}. Exported as a named union so a
|
|
10
|
+
* caller can exhaustively switch on `err.code` and have the compiler point at the case they
|
|
11
|
+
* forgot when a release adds one.
|
|
12
|
+
* @typedef {(typeof codes)[keyof typeof codes]} ErrorCode
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
/** Base for everything this package throws. Never thrown directly. */
|
|
16
|
+
export class TunnelFetchError extends Error {
|
|
17
|
+
/**
|
|
18
|
+
* @param {string} code stable machine-readable code, normally an {@link ErrorCode}. Typed as
|
|
19
|
+
* string because one internal code (`UNEXPECTED_EOF`, from the byte layer) deliberately
|
|
20
|
+
* lives outside the public table.
|
|
21
|
+
* @param {string} message human-readable, must name concrete values
|
|
22
|
+
* @param {Record<string, unknown>} [detail]
|
|
23
|
+
*/
|
|
24
|
+
constructor(code, message, detail) {
|
|
25
|
+
super(message);
|
|
26
|
+
this.name = new.target.name;
|
|
27
|
+
this.code = code;
|
|
28
|
+
if (detail) this.detail = detail;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** Proxy handshake failed: CONNECT refused, SOCKS5 rejected, auth required. */
|
|
33
|
+
export class ProxyError extends TunnelFetchError {}
|
|
34
|
+
|
|
35
|
+
/** HTTP/1.1 wire format: malformed message, ambiguous framing, truncated body. */
|
|
36
|
+
export class HttpError extends TunnelFetchError {}
|
|
37
|
+
|
|
38
|
+
/** TLS record layer, handshake, or a peer alert. */
|
|
39
|
+
export class TlsError extends TunnelFetchError {}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* HTTP/2 framing, streams, flow control, HPACK, or a peer GOAWAY/RST_STREAM. Separate from
|
|
43
|
+
* HttpError because the wire format and its failure modes are entirely different: an HTTP/1.1
|
|
44
|
+
* message is text framed by lengths and CRLFs, an HTTP/2 stream is a state machine over binary
|
|
45
|
+
* frames sharing one connection, and a caller distinguishing "the /1.1 parser choked" from "an
|
|
46
|
+
* h2 stream was reset" wants two codes, not one.
|
|
47
|
+
*/
|
|
48
|
+
export class Http2Error extends TunnelFetchError {}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* The peer offered something we deliberately do not implement.
|
|
52
|
+
* Separate from TlsError so "we cannot talk to this server" is distinguishable from
|
|
53
|
+
* "the connection broke", which is the difference between a feature request and a bug report.
|
|
54
|
+
*/
|
|
55
|
+
export class TlsUnsupportedError extends TlsError {}
|
|
56
|
+
|
|
57
|
+
/** Certificate chain, trust anchor, name matching, validity, or constraint failure. */
|
|
58
|
+
export class CertificateError extends TunnelFetchError {}
|
|
59
|
+
|
|
60
|
+
/** A deadline elapsed. `phase` says which one, because the fix differs per phase. */
|
|
61
|
+
export class TimeoutError extends TunnelFetchError {}
|
|
62
|
+
|
|
63
|
+
/** A configured limit was exceeded (response size, header size, redirect count). */
|
|
64
|
+
export class LimitError extends TunnelFetchError {}
|
|
65
|
+
|
|
66
|
+
/** The caller asked for something the chosen transport cannot honour. */
|
|
67
|
+
export class ConfigError extends TunnelFetchError {}
|
|
68
|
+
|
|
69
|
+
export const codes = /** @type {const} */ ({
|
|
70
|
+
// proxy
|
|
71
|
+
PROXY_CONNECT_REFUSED: 'PROXY_CONNECT_REFUSED',
|
|
72
|
+
PROXY_AUTH_REQUIRED: 'PROXY_AUTH_REQUIRED',
|
|
73
|
+
PROXY_AUTH_FAILED: 'PROXY_AUTH_FAILED',
|
|
74
|
+
PROXY_PROTOCOL: 'PROXY_PROTOCOL',
|
|
75
|
+
PROXY_UNREACHABLE: 'PROXY_UNREACHABLE',
|
|
76
|
+
SOCKS5_NO_ACCEPTABLE_AUTH: 'SOCKS5_NO_ACCEPTABLE_AUTH',
|
|
77
|
+
SOCKS5_REPLY: 'SOCKS5_REPLY',
|
|
78
|
+
SOCKS5_ADDR_TYPE: 'SOCKS5_ADDR_TYPE',
|
|
79
|
+
|
|
80
|
+
// http/1.1
|
|
81
|
+
HTTP_REQUEST_LINE: 'HTTP_REQUEST_LINE',
|
|
82
|
+
HTTP_STATUS_LINE: 'HTTP_STATUS_LINE',
|
|
83
|
+
HTTP_HEADER: 'HTTP_HEADER',
|
|
84
|
+
HTTP_FRAMING_AMBIGUOUS: 'HTTP_FRAMING_AMBIGUOUS',
|
|
85
|
+
HTTP_CHUNK: 'HTTP_CHUNK',
|
|
86
|
+
HTTP_BODY_TRUNCATED: 'HTTP_BODY_TRUNCATED',
|
|
87
|
+
HTTP_TRAILER: 'HTTP_TRAILER',
|
|
88
|
+
HTTP_UPGRADE_UNEXPECTED: 'HTTP_UPGRADE_UNEXPECTED',
|
|
89
|
+
|
|
90
|
+
// tls
|
|
91
|
+
TLS_RECORD: 'TLS_RECORD',
|
|
92
|
+
TLS_ALERT: 'TLS_ALERT',
|
|
93
|
+
TLS_HANDSHAKE: 'TLS_HANDSHAKE',
|
|
94
|
+
TLS_TRUNCATED: 'TLS_TRUNCATED',
|
|
95
|
+
TLS_VERSION_UNSUPPORTED: 'TLS_VERSION_UNSUPPORTED',
|
|
96
|
+
TLS_CIPHER_UNSUPPORTED: 'TLS_CIPHER_UNSUPPORTED',
|
|
97
|
+
TLS_GROUP_UNSUPPORTED: 'TLS_GROUP_UNSUPPORTED',
|
|
98
|
+
TLS_SIGALG_UNSUPPORTED: 'TLS_SIGALG_UNSUPPORTED',
|
|
99
|
+
TLS_EXTENSION_UNSUPPORTED: 'TLS_EXTENSION_UNSUPPORTED',
|
|
100
|
+
TLS_ALPN: 'TLS_ALPN',
|
|
101
|
+
// Session resumption. TLS_PSK is a pre_shared_key negotiation violation (a selected identity
|
|
102
|
+
// that was not offered, a suite whose hash disagrees with the PSK, a selection with no offer);
|
|
103
|
+
// TLS_TICKET is an unusable NewSessionTicket or resumption ticket. Split because the first
|
|
104
|
+
// means the server broke the protocol mid-handshake and the second means a credential was bad.
|
|
105
|
+
TLS_PSK: 'TLS_PSK',
|
|
106
|
+
TLS_TICKET: 'TLS_TICKET',
|
|
107
|
+
|
|
108
|
+
// http/2 (RFC 9113) and HPACK (RFC 7541). Every one names the concrete wire value that
|
|
109
|
+
// triggered it — a frame type, a stream id, a table index — because "h2 error" alone decides
|
|
110
|
+
// nothing about whether a peer is broken, hostile, or speaking a feature we declined.
|
|
111
|
+
HTTP2_PROTOCOL: 'HTTP2_PROTOCOL', // generic connection PROTOCOL_ERROR: preface, frame on wrong stream, ...
|
|
112
|
+
HTTP2_FRAME_SIZE: 'HTTP2_FRAME_SIZE', // a frame length the type or our SETTINGS forbids
|
|
113
|
+
HTTP2_SETTINGS: 'HTTP2_SETTINGS', // malformed SETTINGS, or a value out of its legal range
|
|
114
|
+
HTTP2_FLOW_CONTROL: 'HTTP2_FLOW_CONTROL', // a window overflowed 2^31-1, or the peer overran ours
|
|
115
|
+
HTTP2_STREAM_STATE: 'HTTP2_STREAM_STATE', // a frame illegal for the stream's current state
|
|
116
|
+
HTTP2_STREAM_CLOSED: 'HTTP2_STREAM_CLOSED', // the peer RST_STREAM'd, or closed a stream we were using
|
|
117
|
+
HTTP2_GOAWAY: 'HTTP2_GOAWAY', // the peer is going away and did not (or will not) serve this stream
|
|
118
|
+
HTTP2_COMPRESSION: 'HTTP2_COMPRESSION', // HPACK: bad index, bad integer, invalid Huffman padding, ...
|
|
119
|
+
HTTP2_HEADER: 'HTTP2_HEADER', // a decoded header list that h2 forbids (uppercase, bad pseudo-header)
|
|
120
|
+
HTTP2_PUSH_UNEXPECTED: 'HTTP2_PUSH_UNEXPECTED', // PUSH_PROMISE despite our SETTINGS_ENABLE_PUSH = 0
|
|
121
|
+
HTTP2_TRAILER: 'HTTP2_TRAILER', // a trailing header block that is malformed or carries pseudo-headers
|
|
122
|
+
|
|
123
|
+
// trust
|
|
124
|
+
CERT_PARSE: 'CERT_PARSE',
|
|
125
|
+
CERT_CHAIN_INCOMPLETE: 'CERT_CHAIN_INCOMPLETE',
|
|
126
|
+
CERT_UNTRUSTED_ROOT: 'CERT_UNTRUSTED_ROOT',
|
|
127
|
+
CERT_SIGNATURE_INVALID: 'CERT_SIGNATURE_INVALID',
|
|
128
|
+
CERT_SIGNATURE_WEAK: 'CERT_SIGNATURE_WEAK',
|
|
129
|
+
CERT_SIGNATURE_UNSUPPORTED: 'CERT_SIGNATURE_UNSUPPORTED',
|
|
130
|
+
CERT_EXPIRED: 'CERT_EXPIRED',
|
|
131
|
+
CERT_NOT_YET_VALID: 'CERT_NOT_YET_VALID',
|
|
132
|
+
CERT_NAME_MISMATCH: 'CERT_NAME_MISMATCH',
|
|
133
|
+
CERT_CONSTRAINT: 'CERT_CONSTRAINT',
|
|
134
|
+
CERT_PIN_MISMATCH: 'CERT_PIN_MISMATCH',
|
|
135
|
+
|
|
136
|
+
// revocation, via stapled OCSP (RFC 6960 over RFC 6066/8446 status_request)
|
|
137
|
+
OCSP_PARSE: 'OCSP_PARSE',
|
|
138
|
+
OCSP_UNVERIFIED: 'OCSP_UNVERIFIED',
|
|
139
|
+
OCSP_MISMATCH: 'OCSP_MISMATCH',
|
|
140
|
+
OCSP_STALE: 'OCSP_STALE',
|
|
141
|
+
OCSP_REVOKED: 'OCSP_REVOKED',
|
|
142
|
+
OCSP_UNKNOWN: 'OCSP_UNKNOWN',
|
|
143
|
+
OCSP_REQUIRED: 'OCSP_REQUIRED',
|
|
144
|
+
|
|
145
|
+
// client semantics
|
|
146
|
+
HTTP_CONTENT_ENCODING: 'HTTP_CONTENT_ENCODING',
|
|
147
|
+
HTTP_CHARSET: 'HTTP_CHARSET',
|
|
148
|
+
REDIRECT_INVALID_LOCATION: 'REDIRECT_INVALID_LOCATION',
|
|
149
|
+
REDIRECT_LOOP: 'REDIRECT_LOOP',
|
|
150
|
+
REDIRECT_SCHEME: 'REDIRECT_SCHEME',
|
|
151
|
+
COOKIE_INVALID: 'COOKIE_INVALID',
|
|
152
|
+
POOL_CLOSED: 'POOL_CLOSED',
|
|
153
|
+
CONNECTION_CLOSED: 'CONNECTION_CLOSED',
|
|
154
|
+
|
|
155
|
+
// deadlines and limits
|
|
156
|
+
TIMEOUT_CONNECT: 'TIMEOUT_CONNECT',
|
|
157
|
+
TIMEOUT_HANDSHAKE: 'TIMEOUT_HANDSHAKE',
|
|
158
|
+
TIMEOUT_HEADERS: 'TIMEOUT_HEADERS',
|
|
159
|
+
TIMEOUT_IDLE: 'TIMEOUT_IDLE',
|
|
160
|
+
TIMEOUT_TOTAL: 'TIMEOUT_TOTAL',
|
|
161
|
+
LIMIT_BODY: 'LIMIT_BODY',
|
|
162
|
+
LIMIT_HEADER: 'LIMIT_HEADER',
|
|
163
|
+
LIMIT_REDIRECTS: 'LIMIT_REDIRECTS',
|
|
164
|
+
|
|
165
|
+
// configuration
|
|
166
|
+
CONFIG_UNSATISFIABLE: 'CONFIG_UNSATISFIABLE',
|
|
167
|
+
CONFIG_INVALID: 'CONFIG_INVALID',
|
|
168
|
+
});
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* Format a byte as 0xNN, for error messages that must name a concrete wire value.
|
|
172
|
+
* @param {number} n
|
|
173
|
+
* @returns {string}
|
|
174
|
+
*/
|
|
175
|
+
export const hex8 = (n) => `0x${(n & 0xff).toString(16).padStart(2, '0')}`;
|
|
176
|
+
/**
|
|
177
|
+
* Format a 16-bit wire value as 0xNNNN (cipher suites, groups, signature schemes).
|
|
178
|
+
* @param {number} n
|
|
179
|
+
* @returns {string}
|
|
180
|
+
*/
|
|
181
|
+
export const hex16 = (n) => `0x${(n & 0xffff).toString(16).padStart(4, '0')}`;
|
|
@@ -0,0 +1,289 @@
|
|
|
1
|
+
// Chunked transfer-coding decoder (RFC 9112 §7.1).
|
|
2
|
+
//
|
|
3
|
+
// Chunked framing is the only place in HTTP/1.1 where the peer controls both the lengths and
|
|
4
|
+
// the delimiters, which makes it the classic smuggling surface. The rules here are therefore
|
|
5
|
+
// grammar-exact: a chunk size is HEX digits and nothing else (no sign, no leading whitespace,
|
|
6
|
+
// no `0x`), chunk data must be followed by exactly CRLF, and any deviation is an error rather
|
|
7
|
+
// than a resynchronisation attempt. A decoder that "recovers" from a framing error has silently
|
|
8
|
+
// agreed to a different message boundary than the sender — that is how one response's bytes
|
|
9
|
+
// become the next response's head.
|
|
10
|
+
//
|
|
11
|
+
// The data phase is driven purely by the declared length, never by delimiter scanning, so
|
|
12
|
+
// payload bytes that happen to contain CRLF (or another chunk header) are inert.
|
|
13
|
+
|
|
14
|
+
import { HttpError, LimitError, codes } from '../errors.js';
|
|
15
|
+
import { latin1, utf8, UnexpectedEofError } from '../util/bytes.js';
|
|
16
|
+
|
|
17
|
+
const LF = utf8('\n');
|
|
18
|
+
|
|
19
|
+
// RFC 9110 token, for trailer field names. Kept local: importing it from response.js would
|
|
20
|
+
// create a cycle (response.js imports this module for body framing).
|
|
21
|
+
const TOKEN_RE = /^[A-Za-z0-9!#$%&'*+\-.^_`|~]+$/;
|
|
22
|
+
|
|
23
|
+
// chunk-size [chunk-ext]: hex digits, then optionally BWS and a `;`-introduced extension we
|
|
24
|
+
// discard. The alternation is anchored so `+1`, `-1`, ` 1`, `0x1` and a bare trailing space
|
|
25
|
+
// all fail to match instead of parsing as a number.
|
|
26
|
+
const SIZE_LINE_RE = /^([0-9A-Fa-f]+)(?:[ \t]*;.*)?$/;
|
|
27
|
+
|
|
28
|
+
// field-value content: HTAB / SP / VCHAR / obs-text. Excludes NUL, CR, LF and other controls.
|
|
29
|
+
const FIELD_VALUE_RE = /^[\t\x20-\x7e\x80-\xff]*$/;
|
|
30
|
+
|
|
31
|
+
// A chunk-size line has no business being long: 13 hex digits already cover every size a safe
|
|
32
|
+
// integer can hold, and extensions are ignored. The bound stops a peer from streaming an
|
|
33
|
+
// unbounded "extension" that we would otherwise buffer while looking for its CRLF.
|
|
34
|
+
const CHUNK_LINE_MAX = 1024;
|
|
35
|
+
|
|
36
|
+
/** A promise with its settle functions exposed, pre-detached so an unobserved rejection
|
|
37
|
+
* (caller consumed the stream but never awaited `trailers`) cannot crash the process. */
|
|
38
|
+
function deferred() {
|
|
39
|
+
let resolve, reject;
|
|
40
|
+
const promise = new Promise((res, rej) => {
|
|
41
|
+
resolve = res;
|
|
42
|
+
reject = rej;
|
|
43
|
+
});
|
|
44
|
+
promise.catch(() => {});
|
|
45
|
+
return { promise, resolve, reject };
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** Read one CRLF-terminated line for chunk framing, mapping low-level failures to HTTP codes. */
|
|
49
|
+
async function readChunkLine(reader, what) {
|
|
50
|
+
let raw;
|
|
51
|
+
try {
|
|
52
|
+
raw = await reader.readUntil(LF, CHUNK_LINE_MAX, what);
|
|
53
|
+
} catch (e) {
|
|
54
|
+
if (e instanceof UnexpectedEofError) {
|
|
55
|
+
throw new HttpError(codes.HTTP_BODY_TRUNCATED, `stream ended inside ${what}`, {
|
|
56
|
+
what,
|
|
57
|
+
got: e.detail?.got,
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
if (e instanceof LimitError) {
|
|
61
|
+
// readUntil reports LIMIT_HEADER, but an endless chunk-size line is a framing attack,
|
|
62
|
+
// not an oversized header block; keep the code honest.
|
|
63
|
+
throw new HttpError(codes.HTTP_CHUNK, `${what} exceeded ${CHUNK_LINE_MAX} bytes`, { what });
|
|
64
|
+
}
|
|
65
|
+
throw e;
|
|
66
|
+
}
|
|
67
|
+
const line = latin1(raw);
|
|
68
|
+
if (line.length < 2 || !line.endsWith('\r\n')) {
|
|
69
|
+
throw new HttpError(codes.HTTP_CHUNK, `${what} ended with bare LF, expected CRLF`, {
|
|
70
|
+
what,
|
|
71
|
+
line,
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
return line.slice(0, -2);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Read the trailer section: zero or more `name: value` lines, then an empty line.
|
|
79
|
+
* The empty line doubles as the terminal CRLF of the whole chunked body.
|
|
80
|
+
*/
|
|
81
|
+
async function readTrailers(reader, maxTrailerBytes) {
|
|
82
|
+
const trailers = new Headers();
|
|
83
|
+
let left = maxTrailerBytes;
|
|
84
|
+
for (;;) {
|
|
85
|
+
let raw;
|
|
86
|
+
try {
|
|
87
|
+
// The budget shrinks per line so the whole section is bounded, and readUntil's own
|
|
88
|
+
// LIMIT_HEADER is the right code here: trailers are header fields.
|
|
89
|
+
raw = await reader.readUntil(LF, left, 'trailer section');
|
|
90
|
+
} catch (e) {
|
|
91
|
+
if (e instanceof UnexpectedEofError) {
|
|
92
|
+
throw new HttpError(
|
|
93
|
+
codes.HTTP_BODY_TRUNCATED,
|
|
94
|
+
'stream ended inside the trailer section, before the terminal CRLF',
|
|
95
|
+
);
|
|
96
|
+
}
|
|
97
|
+
throw e;
|
|
98
|
+
}
|
|
99
|
+
left -= raw.byteLength;
|
|
100
|
+
const line = latin1(raw);
|
|
101
|
+
if (line.length < 2 || !line.endsWith('\r\n')) {
|
|
102
|
+
throw new HttpError(codes.HTTP_TRAILER, 'trailer line ended with bare LF, expected CRLF', {
|
|
103
|
+
line,
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
const s = line.slice(0, -2);
|
|
107
|
+
if (s === '') return trailers;
|
|
108
|
+
if (s[0] === ' ' || s[0] === '\t') {
|
|
109
|
+
// obs-fold; deprecated, and a folded line is exactly how a smuggled field hides.
|
|
110
|
+
throw new HttpError(codes.HTTP_TRAILER, 'trailer line starts with whitespace (obs-fold)', {
|
|
111
|
+
line: s,
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
const colon = s.indexOf(':');
|
|
115
|
+
if (colon <= 0) {
|
|
116
|
+
throw new HttpError(codes.HTTP_TRAILER, `trailer line ${JSON.stringify(s)} has no name`, {
|
|
117
|
+
line: s,
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
const name = s.slice(0, colon);
|
|
121
|
+
if (!TOKEN_RE.test(name)) {
|
|
122
|
+
throw new HttpError(
|
|
123
|
+
codes.HTTP_TRAILER,
|
|
124
|
+
`trailer name ${JSON.stringify(name)} is not an RFC 9110 token`,
|
|
125
|
+
{ name },
|
|
126
|
+
);
|
|
127
|
+
}
|
|
128
|
+
const value = s.slice(colon + 1).replace(/^[ \t]+|[ \t]+$/g, '');
|
|
129
|
+
if (!FIELD_VALUE_RE.test(value)) {
|
|
130
|
+
throw new HttpError(
|
|
131
|
+
codes.HTTP_TRAILER,
|
|
132
|
+
`trailer ${name} value contains a control byte`,
|
|
133
|
+
{ name, value },
|
|
134
|
+
);
|
|
135
|
+
}
|
|
136
|
+
// Framing cannot be re-litigated here: the body length was already decided, so even a
|
|
137
|
+
// trailer named Content-Length is inert data and is stored, not interpreted.
|
|
138
|
+
trailers.append(name, value);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Every cap is fail-closed: the peer controls chunk sizes and counts, so each one bounds what
|
|
144
|
+
* a hostile sender can make us buffer.
|
|
145
|
+
* @typedef {object} ChunkedOptions
|
|
146
|
+
* @property {number} [maxBytes] total payload cap, default unlimited
|
|
147
|
+
* @property {number} [maxChunkSize] per-chunk cap, default 64 MiB
|
|
148
|
+
* @property {number} [maxTrailerBytes] trailer-section cap, default 8192
|
|
149
|
+
*/
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* Decode a chunked body from `reader`.
|
|
153
|
+
*
|
|
154
|
+
* Returns `{ stream, trailers }`:
|
|
155
|
+
* - `stream`: ReadableStream<Uint8Array> of the decoded payload octets.
|
|
156
|
+
* - `trailers`: Promise<Headers|null>. Resolves with the trailer fields once the terminal
|
|
157
|
+
* chunk and its trailing CRLF have been fully consumed — i.e. the reader is positioned
|
|
158
|
+
* exactly after the body, which is the signal a connection pool needs before reuse.
|
|
159
|
+
* Resolves null if the stream is cancelled before that point (position unknown, do not
|
|
160
|
+
* reuse). Rejects with the same error the stream errors with on a protocol violation.
|
|
161
|
+
*
|
|
162
|
+
* Reads only what the chunked grammar covers; any bytes after the terminal CRLF stay in
|
|
163
|
+
* `reader` for the next message.
|
|
164
|
+
*
|
|
165
|
+
* @param {import('../util/bytes.js').ByteReader} reader
|
|
166
|
+
* @param {ChunkedOptions} [opts]
|
|
167
|
+
* @returns {{ stream: ReadableStream<Uint8Array>, trailers: Promise<Headers | null> }}
|
|
168
|
+
*/
|
|
169
|
+
export function decodeChunked(
|
|
170
|
+
reader,
|
|
171
|
+
{ maxBytes = Infinity, maxChunkSize = 64 * 1024 * 1024, maxTrailerBytes = 8192 } = {},
|
|
172
|
+
) {
|
|
173
|
+
let state = 'size'; // 'size' -> 'data' -> 'data-crlf' -> 'size' ... -> 'done'
|
|
174
|
+
let remaining = 0; // data bytes left in the current chunk
|
|
175
|
+
let total = 0; // payload bytes delivered so far, for maxBytes
|
|
176
|
+
const t = deferred();
|
|
177
|
+
|
|
178
|
+
// Loop until we have enqueued payload or closed; a pull that returns without either would
|
|
179
|
+
// just be re-invoked by the stream machinery, so we save the round-trips.
|
|
180
|
+
async function drive(c) {
|
|
181
|
+
for (;;) {
|
|
182
|
+
if (state === 'size') {
|
|
183
|
+
const line = await readChunkLine(reader, 'chunk size line');
|
|
184
|
+
const m = SIZE_LINE_RE.exec(line);
|
|
185
|
+
if (!m) {
|
|
186
|
+
throw new HttpError(
|
|
187
|
+
codes.HTTP_CHUNK,
|
|
188
|
+
`malformed chunk size line ${JSON.stringify(line)}`,
|
|
189
|
+
{ line },
|
|
190
|
+
);
|
|
191
|
+
}
|
|
192
|
+
const size = parseInt(m[1], 16);
|
|
193
|
+
if (!Number.isSafeInteger(size)) {
|
|
194
|
+
throw new HttpError(
|
|
195
|
+
codes.HTTP_CHUNK,
|
|
196
|
+
`chunk size 0x${m[1]} overflows a safe integer`,
|
|
197
|
+
{ hex: m[1] },
|
|
198
|
+
);
|
|
199
|
+
}
|
|
200
|
+
if (size > maxChunkSize) {
|
|
201
|
+
throw new HttpError(
|
|
202
|
+
codes.HTTP_CHUNK,
|
|
203
|
+
`chunk size ${size} exceeds maxChunkSize ${maxChunkSize}`,
|
|
204
|
+
{ size, limit: maxChunkSize },
|
|
205
|
+
);
|
|
206
|
+
}
|
|
207
|
+
if (size === 0) {
|
|
208
|
+
const trailers = await readTrailers(reader, maxTrailerBytes);
|
|
209
|
+
state = 'done';
|
|
210
|
+
t.resolve(trailers);
|
|
211
|
+
c.close();
|
|
212
|
+
return;
|
|
213
|
+
}
|
|
214
|
+
// Checked at the size line, before any data is buffered, so an over-limit body is
|
|
215
|
+
// rejected for the price of one line rather than maxBytes of transfer.
|
|
216
|
+
if (total + size > maxBytes) {
|
|
217
|
+
throw new LimitError(
|
|
218
|
+
codes.LIMIT_BODY,
|
|
219
|
+
`chunked body reached ${total + size} bytes, over the ${maxBytes} byte limit`,
|
|
220
|
+
{ limit: maxBytes },
|
|
221
|
+
);
|
|
222
|
+
}
|
|
223
|
+
remaining = size;
|
|
224
|
+
state = 'data';
|
|
225
|
+
} else if (state === 'data') {
|
|
226
|
+
// Length-driven, and capped at `remaining` so we can never consume past the chunk
|
|
227
|
+
// into the framing that follows it.
|
|
228
|
+
const chunk = await reader.readSome(Math.min(remaining, 65536));
|
|
229
|
+
if (chunk === null) {
|
|
230
|
+
throw new HttpError(
|
|
231
|
+
codes.HTTP_BODY_TRUNCATED,
|
|
232
|
+
`stream ended with ${remaining} bytes of a chunk unread`,
|
|
233
|
+
{ remaining },
|
|
234
|
+
);
|
|
235
|
+
}
|
|
236
|
+
remaining -= chunk.byteLength;
|
|
237
|
+
total += chunk.byteLength;
|
|
238
|
+
if (remaining === 0) state = 'data-crlf';
|
|
239
|
+
c.enqueue(chunk);
|
|
240
|
+
return;
|
|
241
|
+
} else if (state === 'data-crlf') {
|
|
242
|
+
let two;
|
|
243
|
+
try {
|
|
244
|
+
two = await reader.readExactly(2, 'CRLF after chunk data');
|
|
245
|
+
} catch (e) {
|
|
246
|
+
if (e instanceof UnexpectedEofError) {
|
|
247
|
+
throw new HttpError(codes.HTTP_BODY_TRUNCATED, 'stream ended after chunk data, ' +
|
|
248
|
+
'before its CRLF');
|
|
249
|
+
}
|
|
250
|
+
throw e;
|
|
251
|
+
}
|
|
252
|
+
if (two[0] !== 0x0d || two[1] !== 0x0a) {
|
|
253
|
+
// The sender's size and its actual data disagree. Resynchronising here would mean
|
|
254
|
+
// guessing where the message really ends, so this is fatal.
|
|
255
|
+
const got = `0x${two[0].toString(16)} 0x${two[1].toString(16)}`;
|
|
256
|
+
throw new HttpError(
|
|
257
|
+
codes.HTTP_CHUNK,
|
|
258
|
+
`chunk data not followed by CRLF (got ${got})`,
|
|
259
|
+
{ got: [two[0], two[1]] },
|
|
260
|
+
);
|
|
261
|
+
}
|
|
262
|
+
state = 'size';
|
|
263
|
+
} else {
|
|
264
|
+
c.close();
|
|
265
|
+
return;
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
const stream = new ReadableStream({
|
|
271
|
+
async pull(c) {
|
|
272
|
+
try {
|
|
273
|
+
await drive(c);
|
|
274
|
+
} catch (e) {
|
|
275
|
+
state = 'done';
|
|
276
|
+
t.reject(e);
|
|
277
|
+
c.error(e);
|
|
278
|
+
}
|
|
279
|
+
},
|
|
280
|
+
cancel() {
|
|
281
|
+
// Cancellation abandons the connection at an unknown byte position; `null` tells the
|
|
282
|
+
// caller there are no trailers and, implicitly, that the socket must not be reused.
|
|
283
|
+
state = 'done';
|
|
284
|
+
t.resolve(null);
|
|
285
|
+
},
|
|
286
|
+
});
|
|
287
|
+
|
|
288
|
+
return { stream, trailers: t.promise };
|
|
289
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
// HTTP/1.1 wire layer: serialise request heads, parse response heads, frame bodies.
|
|
2
|
+
//
|
|
3
|
+
// This layer owns message boundaries and nothing else. It does not manage connections, follow
|
|
4
|
+
// redirects, or decode content codings; it guarantees exactly one thing — that the bytes it
|
|
5
|
+
// attributes to a message are precisely the bytes the framing rules assign to it, and that any
|
|
6
|
+
// ambiguity about where a message ends is an error instead of a guess.
|
|
7
|
+
|
|
8
|
+
export { serializeRequestHead } from './request.js';
|
|
9
|
+
export { readResponseHead, bodyFraming, readResponseBody } from './response.js';
|
|
10
|
+
export { decodeChunked } from './chunked.js';
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
// HTTP/1.1 request head serialiser.
|
|
2
|
+
//
|
|
3
|
+
// This is a pure function from a description to bytes. It adds nothing implicitly — no Host, no
|
|
4
|
+
// Content-Length, no Connection — because framing and connection management are decisions that
|
|
5
|
+
// belong to the layer that owns the socket. A serialiser with hidden defaults cannot be tested
|
|
6
|
+
// byte-for-byte, and byte-for-byte tests are the only kind that catch request-splitting bugs.
|
|
7
|
+
//
|
|
8
|
+
// Everything the caller provides is validated against the RFC 9110 grammar before it is allowed
|
|
9
|
+
// anywhere near the wire. A header value containing CR or LF is not "escaped" or "sanitised" —
|
|
10
|
+
// it is rejected, because a value that needs sanitising is an injection attempt, and quietly
|
|
11
|
+
// rewriting it turns a loud failure into a silent vulnerability.
|
|
12
|
+
|
|
13
|
+
import { HttpError, codes } from '../errors.js';
|
|
14
|
+
|
|
15
|
+
// RFC 9110 token: 1*tchar. This is the exact alphabet; anything else in a field name or method
|
|
16
|
+
// is either a typo or a smuggling attempt, and both deserve the same rejection.
|
|
17
|
+
const TOKEN_RE = /^[A-Za-z0-9!#$%&'*+\-.^_`|~]+$/;
|
|
18
|
+
|
|
19
|
+
// request-target: printable ASCII with no whitespace. Covers origin-form (`/path?q`),
|
|
20
|
+
// absolute-form (`http://host/path`, required when talking through a forward proxy for
|
|
21
|
+
// plain http), authority-form (`host:port` for CONNECT) and asterisk-form (`*`).
|
|
22
|
+
// A space or control character in the target splits the request line.
|
|
23
|
+
const TARGET_RE = /^[\x21-\x7e]+$/;
|
|
24
|
+
|
|
25
|
+
// The only versions this layer speaks. Serialising `HTTP/2` framing as text would produce
|
|
26
|
+
// garbage a peer might partially interpret, which is worse than failing here.
|
|
27
|
+
const VERSIONS = new Set(['1.0', '1.1']);
|
|
28
|
+
|
|
29
|
+
/** Encode a string whose char codes are all <= 0xFF as raw octets (inverse of bytes.js latin1). */
|
|
30
|
+
function latin1Bytes(s) {
|
|
31
|
+
const out = new Uint8Array(s.length);
|
|
32
|
+
for (let i = 0; i < s.length; i++) out[i] = s.charCodeAt(i);
|
|
33
|
+
return out;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* A request head as this serialiser consumes it. Nothing optional is invented: what is absent
|
|
38
|
+
* here is absent on the wire.
|
|
39
|
+
* @typedef {object} RequestHead
|
|
40
|
+
* @property {string} method RFC 9110 token, sent verbatim (no case-folding)
|
|
41
|
+
* @property {string} target request-target, already encoded: origin-form (`/path?q`),
|
|
42
|
+
* absolute-form, authority-form (CONNECT) or `*`
|
|
43
|
+
* @property {Headers | Iterable<[string, string]>} [headers]
|
|
44
|
+
* @property {'1.0' | '1.1'} [httpVersion] default '1.1'
|
|
45
|
+
*/
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Serialise a request head: request line, header fields, terminating blank line.
|
|
49
|
+
* Throws HttpError on any input the RFC 9110 grammar rejects — notably CR/LF in a value,
|
|
50
|
+
* which is header injection, never something to sanitise.
|
|
51
|
+
*
|
|
52
|
+
* `headers` may be a WHATWG Headers instance or any iterable of [name, value] pairs.
|
|
53
|
+
* Pair iterables are written in caller order, unsorted — order can matter to real servers.
|
|
54
|
+
* Note that a Headers instance has already lost the caller's order by spec (it iterates
|
|
55
|
+
* lowercased and sorted); we serialise its iteration order as-is.
|
|
56
|
+
*
|
|
57
|
+
* @param {RequestHead} req
|
|
58
|
+
* @returns {Uint8Array}
|
|
59
|
+
*/
|
|
60
|
+
export function serializeRequestHead({ method, target, headers = [], httpVersion = '1.1' }) {
|
|
61
|
+
if (typeof method !== 'string' || !TOKEN_RE.test(method)) {
|
|
62
|
+
throw new HttpError(
|
|
63
|
+
codes.HTTP_REQUEST_LINE,
|
|
64
|
+
`method ${JSON.stringify(method)} is not an RFC 9110 token`,
|
|
65
|
+
{ method },
|
|
66
|
+
);
|
|
67
|
+
}
|
|
68
|
+
if (typeof target !== 'string' || !TARGET_RE.test(target)) {
|
|
69
|
+
throw new HttpError(
|
|
70
|
+
codes.HTTP_REQUEST_LINE,
|
|
71
|
+
`request-target ${JSON.stringify(target)} contains whitespace, control bytes, or ` +
|
|
72
|
+
'non-ASCII; targets must arrive here already encoded',
|
|
73
|
+
{ target },
|
|
74
|
+
);
|
|
75
|
+
}
|
|
76
|
+
if (!VERSIONS.has(httpVersion)) {
|
|
77
|
+
throw new HttpError(
|
|
78
|
+
codes.HTTP_REQUEST_LINE,
|
|
79
|
+
`httpVersion ${JSON.stringify(httpVersion)} is not supported (only 1.0 and 1.1)`,
|
|
80
|
+
{ httpVersion },
|
|
81
|
+
);
|
|
82
|
+
}
|
|
83
|
+
if (headers === null || typeof headers[Symbol.iterator] !== 'function') {
|
|
84
|
+
throw new HttpError(
|
|
85
|
+
codes.HTTP_HEADER,
|
|
86
|
+
'headers must be a Headers instance or an iterable of [name, value] pairs',
|
|
87
|
+
);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
let head = `${method} ${target} HTTP/${httpVersion}\r\n`;
|
|
91
|
+
for (const entry of headers) {
|
|
92
|
+
const name = String(entry[0]);
|
|
93
|
+
const rawValue = String(entry[1]);
|
|
94
|
+
if (!TOKEN_RE.test(name)) {
|
|
95
|
+
throw new HttpError(
|
|
96
|
+
codes.HTTP_HEADER,
|
|
97
|
+
`header name ${JSON.stringify(name)} is not an RFC 9110 token`,
|
|
98
|
+
{ name },
|
|
99
|
+
);
|
|
100
|
+
}
|
|
101
|
+
// The request-splitting defence. Checked on the raw value BEFORE trimming, so a value like
|
|
102
|
+
// 'x\r\n' is rejected rather than quietly repaired into something send-safe.
|
|
103
|
+
if (/[\r\n\0]/.test(rawValue)) {
|
|
104
|
+
throw new HttpError(
|
|
105
|
+
codes.HTTP_HEADER,
|
|
106
|
+
`header ${name} value ${JSON.stringify(rawValue)} contains CR, LF, or NUL ` +
|
|
107
|
+
'(header injection rejected)',
|
|
108
|
+
{ name, value: rawValue },
|
|
109
|
+
);
|
|
110
|
+
}
|
|
111
|
+
// Field values are opaque octets; a char code over 0xFF has no octet representation and
|
|
112
|
+
// guessing an encoding for it would send bytes the caller never wrote.
|
|
113
|
+
for (let i = 0; i < rawValue.length; i++) {
|
|
114
|
+
if (rawValue.charCodeAt(i) > 0xff) {
|
|
115
|
+
throw new HttpError(
|
|
116
|
+
codes.HTTP_HEADER,
|
|
117
|
+
`header ${name} value contains U+${rawValue.codePointAt(i).toString(16)} which is not ` +
|
|
118
|
+
'representable as a single octet',
|
|
119
|
+
{ name, value: rawValue },
|
|
120
|
+
);
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
// Beyond CR/LF/NUL, the RFC 9110 field-value grammar admits only HTAB, SP through '~', and
|
|
124
|
+
// obs-text (0x80-0xFF). The remaining control bytes — 0x01-0x08, VT, FF, 0x0E-0x1F and DEL —
|
|
125
|
+
// are not send-safe: the response parser refuses them on receipt (FIELD_VALUE_RE), and a
|
|
126
|
+
// serialiser laxer than the parser is a hole in the same fail-closed grammar. Reject rather
|
|
127
|
+
// than emit a byte the caller almost certainly did not mean and a proxy may mishandle.
|
|
128
|
+
if (/[\x01-\x08\x0b\x0c\x0e-\x1f\x7f]/.test(rawValue)) {
|
|
129
|
+
throw new HttpError(
|
|
130
|
+
codes.HTTP_HEADER,
|
|
131
|
+
`header ${name} value contains a control byte the RFC 9110 field-value grammar forbids`,
|
|
132
|
+
{ name, value: rawValue },
|
|
133
|
+
);
|
|
134
|
+
}
|
|
135
|
+
// Leading/trailing OWS is meaningless on the wire (the peer must strip it) and Headers
|
|
136
|
+
// instances arrive pre-trimmed; trimming pair-iterable input keeps both paths identical.
|
|
137
|
+
const value = rawValue.replace(/^[ \t]+|[ \t]+$/g, '');
|
|
138
|
+
head += `${name}: ${value}\r\n`;
|
|
139
|
+
}
|
|
140
|
+
head += '\r\n';
|
|
141
|
+
// Every part was validated to char codes <= 0xFF, so latin1 encoding is exact.
|
|
142
|
+
return latin1Bytes(head);
|
|
143
|
+
}
|