undici 7.16.0 → 7.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +53 -1
- package/docs/docs/api/Client.md +1 -0
- package/docs/docs/api/DiagnosticsChannel.md +57 -0
- package/docs/docs/api/Dispatcher.md +86 -0
- package/docs/docs/api/RoundRobinPool.md +145 -0
- package/docs/docs/api/WebSocket.md +21 -0
- package/docs/docs/best-practices/crawling.md +58 -0
- package/index.js +4 -1
- package/lib/api/api-upgrade.js +2 -1
- package/lib/core/connect.js +4 -1
- package/lib/core/diagnostics.js +28 -1
- package/lib/core/symbols.js +3 -0
- package/lib/core/util.js +29 -31
- package/lib/dispatcher/balanced-pool.js +10 -0
- package/lib/dispatcher/client-h1.js +0 -16
- package/lib/dispatcher/client-h2.js +153 -23
- package/lib/dispatcher/client.js +7 -2
- package/lib/dispatcher/dispatcher-base.js +11 -12
- package/lib/dispatcher/h2c-client.js +7 -78
- package/lib/dispatcher/pool-base.js +1 -1
- package/lib/dispatcher/proxy-agent.js +13 -2
- package/lib/dispatcher/round-robin-pool.js +137 -0
- package/lib/encoding/index.js +33 -0
- package/lib/handler/cache-handler.js +84 -27
- package/lib/handler/deduplication-handler.js +216 -0
- package/lib/handler/retry-handler.js +0 -2
- package/lib/interceptor/cache.js +35 -17
- package/lib/interceptor/decompress.js +2 -1
- package/lib/interceptor/deduplicate.js +109 -0
- package/lib/interceptor/dns.js +55 -13
- package/lib/mock/mock-utils.js +1 -2
- package/lib/mock/snapshot-agent.js +11 -5
- package/lib/mock/snapshot-recorder.js +12 -4
- package/lib/mock/snapshot-utils.js +4 -4
- package/lib/util/cache.js +29 -1
- package/lib/util/runtime-features.js +124 -0
- package/lib/web/cookies/parse.js +1 -1
- package/lib/web/fetch/body.js +29 -39
- package/lib/web/fetch/data-url.js +12 -160
- package/lib/web/fetch/formdata-parser.js +204 -127
- package/lib/web/fetch/index.js +18 -6
- package/lib/web/fetch/request.js +6 -0
- package/lib/web/fetch/response.js +2 -3
- package/lib/web/fetch/util.js +2 -65
- package/lib/web/infra/index.js +229 -0
- package/lib/web/subresource-integrity/subresource-integrity.js +6 -5
- package/lib/web/webidl/index.js +4 -2
- package/lib/web/websocket/connection.js +31 -21
- package/lib/web/websocket/frame.js +9 -15
- package/lib/web/websocket/stream/websocketstream.js +1 -1
- package/lib/web/websocket/util.js +2 -1
- package/package.json +5 -4
- package/types/agent.d.ts +1 -1
- package/types/api.d.ts +2 -2
- package/types/balanced-pool.d.ts +2 -1
- package/types/cache-interceptor.d.ts +1 -0
- package/types/client.d.ts +1 -1
- package/types/connector.d.ts +2 -2
- package/types/diagnostics-channel.d.ts +2 -2
- package/types/dispatcher.d.ts +12 -12
- package/types/fetch.d.ts +4 -4
- package/types/formdata.d.ts +1 -1
- package/types/h2c-client.d.ts +1 -1
- package/types/index.d.ts +9 -1
- package/types/interceptors.d.ts +36 -2
- package/types/pool.d.ts +1 -1
- package/types/readable.d.ts +2 -2
- package/types/round-robin-pool.d.ts +41 -0
- package/types/websocket.d.ts +9 -9
package/lib/web/fetch/body.js
CHANGED
|
@@ -5,8 +5,7 @@ const {
|
|
|
5
5
|
ReadableStreamFrom,
|
|
6
6
|
readableStreamClose,
|
|
7
7
|
fullyReadBody,
|
|
8
|
-
extractMimeType
|
|
9
|
-
utf8DecodeBytes
|
|
8
|
+
extractMimeType
|
|
10
9
|
} = require('./util')
|
|
11
10
|
const { FormData, setFormDataState } = require('./formdata')
|
|
12
11
|
const { webidl } = require('../webidl')
|
|
@@ -16,15 +15,13 @@ const { isArrayBuffer } = require('node:util/types')
|
|
|
16
15
|
const { serializeAMimeType } = require('./data-url')
|
|
17
16
|
const { multipartFormDataParser } = require('./formdata-parser')
|
|
18
17
|
const { createDeferredPromise } = require('../../util/promise')
|
|
18
|
+
const { parseJSONFromBytes } = require('../infra')
|
|
19
|
+
const { utf8DecodeBytes } = require('../../encoding')
|
|
20
|
+
const { runtimeFeatures } = require('../../util/runtime-features.js')
|
|
19
21
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
const crypto = require('node:crypto')
|
|
24
|
-
random = (max) => crypto.randomInt(0, max)
|
|
25
|
-
} catch {
|
|
26
|
-
random = (max) => Math.floor(Math.random() * max)
|
|
27
|
-
}
|
|
22
|
+
const random = runtimeFeatures.has('crypto')
|
|
23
|
+
? require('node:crypto').randomInt
|
|
24
|
+
: (max) => Math.floor(Math.random() * max)
|
|
28
25
|
|
|
29
26
|
const textEncoder = new TextEncoder()
|
|
30
27
|
function noop () {}
|
|
@@ -225,32 +222,33 @@ function extractBody (object, keepalive = false) {
|
|
|
225
222
|
// Run action.
|
|
226
223
|
let iterator
|
|
227
224
|
stream = new ReadableStream({
|
|
228
|
-
|
|
225
|
+
start () {
|
|
229
226
|
iterator = action(object)[Symbol.asyncIterator]()
|
|
230
227
|
},
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
228
|
+
pull (controller) {
|
|
229
|
+
return iterator.next().then(({ value, done }) => {
|
|
230
|
+
if (done) {
|
|
231
|
+
// When running action is done, close stream.
|
|
232
|
+
queueMicrotask(() => {
|
|
233
|
+
controller.close()
|
|
234
|
+
controller.byobRequest?.respond(0)
|
|
235
|
+
})
|
|
236
|
+
} else {
|
|
237
|
+
// Whenever one or more bytes are available and stream is not errored,
|
|
238
|
+
// enqueue a Uint8Array wrapping an ArrayBuffer containing the available
|
|
239
|
+
// bytes into stream.
|
|
240
|
+
if (!isErrored(stream)) {
|
|
241
|
+
const buffer = new Uint8Array(value)
|
|
242
|
+
if (buffer.byteLength) {
|
|
243
|
+
controller.enqueue(buffer)
|
|
244
|
+
}
|
|
247
245
|
}
|
|
248
246
|
}
|
|
249
|
-
|
|
250
|
-
|
|
247
|
+
return controller.desiredSize > 0
|
|
248
|
+
})
|
|
251
249
|
},
|
|
252
|
-
|
|
253
|
-
|
|
250
|
+
cancel (reason) {
|
|
251
|
+
return iterator.return()
|
|
254
252
|
},
|
|
255
253
|
type: 'bytes'
|
|
256
254
|
})
|
|
@@ -496,14 +494,6 @@ function bodyUnusable (object) {
|
|
|
496
494
|
return body != null && (body.stream.locked || util.isDisturbed(body.stream))
|
|
497
495
|
}
|
|
498
496
|
|
|
499
|
-
/**
|
|
500
|
-
* @see https://infra.spec.whatwg.org/#parse-json-bytes-to-a-javascript-value
|
|
501
|
-
* @param {Uint8Array} bytes
|
|
502
|
-
*/
|
|
503
|
-
function parseJSONFromBytes (bytes) {
|
|
504
|
-
return JSON.parse(utf8DecodeBytes(bytes))
|
|
505
|
-
}
|
|
506
|
-
|
|
507
497
|
/**
|
|
508
498
|
* @see https://fetch.spec.whatwg.org/#concept-body-mime-type
|
|
509
499
|
* @param {any} requestOrResponse internal state
|
|
@@ -1,19 +1,20 @@
|
|
|
1
1
|
'use strict'
|
|
2
2
|
|
|
3
3
|
const assert = require('node:assert')
|
|
4
|
+
const { forgivingBase64, collectASequenceOfCodePoints, collectASequenceOfCodePointsFast, isomorphicDecode, removeASCIIWhitespace, removeChars } = require('../infra')
|
|
4
5
|
|
|
5
6
|
const encoder = new TextEncoder()
|
|
6
7
|
|
|
7
8
|
/**
|
|
8
9
|
* @see https://mimesniff.spec.whatwg.org/#http-token-code-point
|
|
9
10
|
*/
|
|
10
|
-
const HTTP_TOKEN_CODEPOINTS = /^[
|
|
11
|
-
const HTTP_WHITESPACE_REGEX = /[\u000A\u000D\u0009\u0020]/ // eslint-disable-line
|
|
12
|
-
|
|
11
|
+
const HTTP_TOKEN_CODEPOINTS = /^[-!#$%&'*+.^_|~A-Za-z0-9]+$/u
|
|
12
|
+
const HTTP_WHITESPACE_REGEX = /[\u000A\u000D\u0009\u0020]/u // eslint-disable-line
|
|
13
|
+
|
|
13
14
|
/**
|
|
14
15
|
* @see https://mimesniff.spec.whatwg.org/#http-quoted-string-token-code-point
|
|
15
16
|
*/
|
|
16
|
-
const HTTP_QUOTED_STRING_TOKENS = /^[\u0009\u0020-\u007E\u0080-\u00FF]+$/ // eslint-disable-line
|
|
17
|
+
const HTTP_QUOTED_STRING_TOKENS = /^[\u0009\u0020-\u007E\u0080-\u00FF]+$/u // eslint-disable-line
|
|
17
18
|
|
|
18
19
|
// https://fetch.spec.whatwg.org/#data-url-processor
|
|
19
20
|
/** @param {URL} dataURL */
|
|
@@ -68,7 +69,7 @@ function dataURLProcessor (dataURL) {
|
|
|
68
69
|
// 11. If mimeType ends with U+003B (;), followed by
|
|
69
70
|
// zero or more U+0020 SPACE, followed by an ASCII
|
|
70
71
|
// case-insensitive match for "base64", then:
|
|
71
|
-
if (/;(
|
|
72
|
+
if (/;(?:\u0020*)base64$/ui.test(mimeType)) {
|
|
72
73
|
// 1. Let stringBody be the isomorphic decode of body.
|
|
73
74
|
const stringBody = isomorphicDecode(body)
|
|
74
75
|
|
|
@@ -86,7 +87,7 @@ function dataURLProcessor (dataURL) {
|
|
|
86
87
|
|
|
87
88
|
// 5. Remove trailing U+0020 SPACE code points from mimeType,
|
|
88
89
|
// if any.
|
|
89
|
-
mimeType = mimeType.replace(/(\u0020)
|
|
90
|
+
mimeType = mimeType.replace(/(\u0020+)$/u, '')
|
|
90
91
|
|
|
91
92
|
// 6. Remove the last U+003B (;) code point from mimeType.
|
|
92
93
|
mimeType = mimeType.slice(0, -1)
|
|
@@ -136,49 +137,6 @@ function URLSerializer (url, excludeFragment = false) {
|
|
|
136
137
|
return serialized
|
|
137
138
|
}
|
|
138
139
|
|
|
139
|
-
// https://infra.spec.whatwg.org/#collect-a-sequence-of-code-points
|
|
140
|
-
/**
|
|
141
|
-
* @param {(char: string) => boolean} condition
|
|
142
|
-
* @param {string} input
|
|
143
|
-
* @param {{ position: number }} position
|
|
144
|
-
*/
|
|
145
|
-
function collectASequenceOfCodePoints (condition, input, position) {
|
|
146
|
-
// 1. Let result be the empty string.
|
|
147
|
-
let result = ''
|
|
148
|
-
|
|
149
|
-
// 2. While position doesn’t point past the end of input and the
|
|
150
|
-
// code point at position within input meets the condition condition:
|
|
151
|
-
while (position.position < input.length && condition(input[position.position])) {
|
|
152
|
-
// 1. Append that code point to the end of result.
|
|
153
|
-
result += input[position.position]
|
|
154
|
-
|
|
155
|
-
// 2. Advance position by 1.
|
|
156
|
-
position.position++
|
|
157
|
-
}
|
|
158
|
-
|
|
159
|
-
// 3. Return result.
|
|
160
|
-
return result
|
|
161
|
-
}
|
|
162
|
-
|
|
163
|
-
/**
|
|
164
|
-
* A faster collectASequenceOfCodePoints that only works when comparing a single character.
|
|
165
|
-
* @param {string} char
|
|
166
|
-
* @param {string} input
|
|
167
|
-
* @param {{ position: number }} position
|
|
168
|
-
*/
|
|
169
|
-
function collectASequenceOfCodePointsFast (char, input, position) {
|
|
170
|
-
const idx = input.indexOf(char, position.position)
|
|
171
|
-
const start = position.position
|
|
172
|
-
|
|
173
|
-
if (idx === -1) {
|
|
174
|
-
position.position = input.length
|
|
175
|
-
return input.slice(start)
|
|
176
|
-
}
|
|
177
|
-
|
|
178
|
-
position.position = idx
|
|
179
|
-
return input.slice(start, position.position)
|
|
180
|
-
}
|
|
181
|
-
|
|
182
140
|
// https://url.spec.whatwg.org/#string-percent-decode
|
|
183
141
|
/** @param {string} input */
|
|
184
142
|
function stringPercentDecode (input) {
|
|
@@ -219,8 +177,9 @@ function percentDecode (input) {
|
|
|
219
177
|
/** @type {Uint8Array} */
|
|
220
178
|
const output = new Uint8Array(length)
|
|
221
179
|
let j = 0
|
|
180
|
+
let i = 0
|
|
222
181
|
// 2. For each byte byte in input:
|
|
223
|
-
|
|
182
|
+
while (i < length) {
|
|
224
183
|
const byte = input[i]
|
|
225
184
|
|
|
226
185
|
// 1. If byte is not 0x25 (%), then append byte to output.
|
|
@@ -248,6 +207,7 @@ function percentDecode (input) {
|
|
|
248
207
|
// 3. Skip the next two bytes in input.
|
|
249
208
|
i += 2
|
|
250
209
|
}
|
|
210
|
+
++i
|
|
251
211
|
}
|
|
252
212
|
|
|
253
213
|
// 3. Return output.
|
|
@@ -427,45 +387,6 @@ function parseMIMEType (input) {
|
|
|
427
387
|
return mimeType
|
|
428
388
|
}
|
|
429
389
|
|
|
430
|
-
// https://infra.spec.whatwg.org/#forgiving-base64-decode
|
|
431
|
-
/** @param {string} data */
|
|
432
|
-
function forgivingBase64 (data) {
|
|
433
|
-
// 1. Remove all ASCII whitespace from data.
|
|
434
|
-
data = data.replace(ASCII_WHITESPACE_REPLACE_REGEX, '')
|
|
435
|
-
|
|
436
|
-
let dataLength = data.length
|
|
437
|
-
// 2. If data’s code point length divides by 4 leaving
|
|
438
|
-
// no remainder, then:
|
|
439
|
-
if (dataLength % 4 === 0) {
|
|
440
|
-
// 1. If data ends with one or two U+003D (=) code points,
|
|
441
|
-
// then remove them from data.
|
|
442
|
-
if (data.charCodeAt(dataLength - 1) === 0x003D) {
|
|
443
|
-
--dataLength
|
|
444
|
-
if (data.charCodeAt(dataLength - 1) === 0x003D) {
|
|
445
|
-
--dataLength
|
|
446
|
-
}
|
|
447
|
-
}
|
|
448
|
-
}
|
|
449
|
-
|
|
450
|
-
// 3. If data’s code point length divides by 4 leaving
|
|
451
|
-
// a remainder of 1, then return failure.
|
|
452
|
-
if (dataLength % 4 === 1) {
|
|
453
|
-
return 'failure'
|
|
454
|
-
}
|
|
455
|
-
|
|
456
|
-
// 4. If data contains a code point that is not one of
|
|
457
|
-
// U+002B (+)
|
|
458
|
-
// U+002F (/)
|
|
459
|
-
// ASCII alphanumeric
|
|
460
|
-
// then return failure.
|
|
461
|
-
if (/[^+/0-9A-Za-z]/.test(data.length === dataLength ? data : data.substring(0, dataLength))) {
|
|
462
|
-
return 'failure'
|
|
463
|
-
}
|
|
464
|
-
|
|
465
|
-
const buffer = Buffer.from(data, 'base64')
|
|
466
|
-
return new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength)
|
|
467
|
-
}
|
|
468
|
-
|
|
469
390
|
// https://fetch.spec.whatwg.org/#collect-an-http-quoted-string
|
|
470
391
|
// tests: https://fetch.spec.whatwg.org/#example-http-quoted-string
|
|
471
392
|
/**
|
|
@@ -572,7 +493,7 @@ function serializeAMimeType (mimeType) {
|
|
|
572
493
|
if (!HTTP_TOKEN_CODEPOINTS.test(value)) {
|
|
573
494
|
// 1. Precede each occurrence of U+0022 (") or
|
|
574
495
|
// U+005C (\) in value with U+005C (\).
|
|
575
|
-
value = value.replace(/
|
|
496
|
+
value = value.replace(/[\\"]/ug, '\\$&')
|
|
576
497
|
|
|
577
498
|
// 2. Prepend U+0022 (") to value.
|
|
578
499
|
value = '"' + value
|
|
@@ -608,71 +529,6 @@ function removeHTTPWhitespace (str, leading = true, trailing = true) {
|
|
|
608
529
|
return removeChars(str, leading, trailing, isHTTPWhiteSpace)
|
|
609
530
|
}
|
|
610
531
|
|
|
611
|
-
/**
|
|
612
|
-
* @see https://infra.spec.whatwg.org/#ascii-whitespace
|
|
613
|
-
* @param {number} char
|
|
614
|
-
*/
|
|
615
|
-
function isASCIIWhitespace (char) {
|
|
616
|
-
// "\r\n\t\f "
|
|
617
|
-
return char === 0x00d || char === 0x00a || char === 0x009 || char === 0x00c || char === 0x020
|
|
618
|
-
}
|
|
619
|
-
|
|
620
|
-
/**
|
|
621
|
-
* @see https://infra.spec.whatwg.org/#strip-leading-and-trailing-ascii-whitespace
|
|
622
|
-
* @param {string} str
|
|
623
|
-
* @param {boolean} [leading=true]
|
|
624
|
-
* @param {boolean} [trailing=true]
|
|
625
|
-
*/
|
|
626
|
-
function removeASCIIWhitespace (str, leading = true, trailing = true) {
|
|
627
|
-
return removeChars(str, leading, trailing, isASCIIWhitespace)
|
|
628
|
-
}
|
|
629
|
-
|
|
630
|
-
/**
|
|
631
|
-
* @param {string} str
|
|
632
|
-
* @param {boolean} leading
|
|
633
|
-
* @param {boolean} trailing
|
|
634
|
-
* @param {(charCode: number) => boolean} predicate
|
|
635
|
-
* @returns
|
|
636
|
-
*/
|
|
637
|
-
function removeChars (str, leading, trailing, predicate) {
|
|
638
|
-
let lead = 0
|
|
639
|
-
let trail = str.length - 1
|
|
640
|
-
|
|
641
|
-
if (leading) {
|
|
642
|
-
while (lead < str.length && predicate(str.charCodeAt(lead))) lead++
|
|
643
|
-
}
|
|
644
|
-
|
|
645
|
-
if (trailing) {
|
|
646
|
-
while (trail > 0 && predicate(str.charCodeAt(trail))) trail--
|
|
647
|
-
}
|
|
648
|
-
|
|
649
|
-
return lead === 0 && trail === str.length - 1 ? str : str.slice(lead, trail + 1)
|
|
650
|
-
}
|
|
651
|
-
|
|
652
|
-
/**
|
|
653
|
-
* @see https://infra.spec.whatwg.org/#isomorphic-decode
|
|
654
|
-
* @param {Uint8Array} input
|
|
655
|
-
* @returns {string}
|
|
656
|
-
*/
|
|
657
|
-
function isomorphicDecode (input) {
|
|
658
|
-
// 1. To isomorphic decode a byte sequence input, return a string whose code point
|
|
659
|
-
// length is equal to input’s length and whose code points have the same values
|
|
660
|
-
// as the values of input’s bytes, in the same order.
|
|
661
|
-
const length = input.length
|
|
662
|
-
if ((2 << 15) - 1 > length) {
|
|
663
|
-
return String.fromCharCode.apply(null, input)
|
|
664
|
-
}
|
|
665
|
-
let result = ''; let i = 0
|
|
666
|
-
let addition = (2 << 15) - 1
|
|
667
|
-
while (i < length) {
|
|
668
|
-
if (i + addition > length) {
|
|
669
|
-
addition = length - i
|
|
670
|
-
}
|
|
671
|
-
result += String.fromCharCode.apply(null, input.subarray(i, i += addition))
|
|
672
|
-
}
|
|
673
|
-
return result
|
|
674
|
-
}
|
|
675
|
-
|
|
676
532
|
/**
|
|
677
533
|
* @see https://mimesniff.spec.whatwg.org/#minimize-a-supported-mime-type
|
|
678
534
|
* @param {Exclude<ReturnType<typeof parseMIMEType>, 'failure'>} mimeType
|
|
@@ -730,15 +586,11 @@ function minimizeSupportedMimeType (mimeType) {
|
|
|
730
586
|
module.exports = {
|
|
731
587
|
dataURLProcessor,
|
|
732
588
|
URLSerializer,
|
|
733
|
-
collectASequenceOfCodePoints,
|
|
734
|
-
collectASequenceOfCodePointsFast,
|
|
735
589
|
stringPercentDecode,
|
|
736
590
|
parseMIMEType,
|
|
737
591
|
collectAnHTTPQuotedString,
|
|
738
592
|
serializeAMimeType,
|
|
739
|
-
removeChars,
|
|
740
593
|
removeHTTPWhitespace,
|
|
741
594
|
minimizeSupportedMimeType,
|
|
742
|
-
HTTP_TOKEN_CODEPOINTS
|
|
743
|
-
isomorphicDecode
|
|
595
|
+
HTTP_TOKEN_CODEPOINTS
|
|
744
596
|
}
|