undici 7.15.0 → 7.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/README.md +48 -2
  2. package/docs/docs/api/Agent.md +1 -0
  3. package/docs/docs/api/Client.md +1 -0
  4. package/docs/docs/api/DiagnosticsChannel.md +57 -0
  5. package/docs/docs/api/Dispatcher.md +86 -0
  6. package/docs/docs/api/Errors.md +0 -1
  7. package/docs/docs/api/RoundRobinPool.md +145 -0
  8. package/docs/docs/api/WebSocket.md +21 -0
  9. package/docs/docs/best-practices/crawling.md +58 -0
  10. package/index-fetch.js +2 -2
  11. package/index.js +8 -9
  12. package/lib/api/api-request.js +22 -8
  13. package/lib/api/api-upgrade.js +2 -1
  14. package/lib/api/readable.js +7 -5
  15. package/lib/core/connect.js +4 -1
  16. package/lib/core/diagnostics.js +28 -1
  17. package/lib/core/errors.js +217 -13
  18. package/lib/core/request.js +5 -1
  19. package/lib/core/symbols.js +3 -0
  20. package/lib/core/util.js +61 -41
  21. package/lib/dispatcher/agent.js +19 -7
  22. package/lib/dispatcher/balanced-pool.js +10 -0
  23. package/lib/dispatcher/client-h1.js +18 -23
  24. package/lib/dispatcher/client-h2.js +166 -26
  25. package/lib/dispatcher/client.js +64 -59
  26. package/lib/dispatcher/dispatcher-base.js +20 -16
  27. package/lib/dispatcher/env-http-proxy-agent.js +12 -16
  28. package/lib/dispatcher/fixed-queue.js +15 -39
  29. package/lib/dispatcher/h2c-client.js +7 -78
  30. package/lib/dispatcher/pool-base.js +60 -43
  31. package/lib/dispatcher/pool.js +2 -2
  32. package/lib/dispatcher/proxy-agent.js +27 -11
  33. package/lib/dispatcher/round-robin-pool.js +137 -0
  34. package/lib/encoding/index.js +33 -0
  35. package/lib/global.js +19 -1
  36. package/lib/handler/cache-handler.js +84 -27
  37. package/lib/handler/deduplication-handler.js +216 -0
  38. package/lib/handler/retry-handler.js +0 -2
  39. package/lib/interceptor/cache.js +94 -15
  40. package/lib/interceptor/decompress.js +2 -1
  41. package/lib/interceptor/deduplicate.js +109 -0
  42. package/lib/interceptor/dns.js +55 -13
  43. package/lib/mock/mock-agent.js +4 -4
  44. package/lib/mock/mock-errors.js +10 -0
  45. package/lib/mock/mock-utils.js +13 -12
  46. package/lib/mock/snapshot-agent.js +11 -5
  47. package/lib/mock/snapshot-recorder.js +12 -4
  48. package/lib/mock/snapshot-utils.js +4 -4
  49. package/lib/util/cache.js +29 -1
  50. package/lib/util/date.js +534 -140
  51. package/lib/util/runtime-features.js +124 -0
  52. package/lib/web/cookies/index.js +1 -1
  53. package/lib/web/cookies/parse.js +1 -1
  54. package/lib/web/eventsource/eventsource-stream.js +2 -2
  55. package/lib/web/eventsource/eventsource.js +34 -29
  56. package/lib/web/eventsource/util.js +1 -9
  57. package/lib/web/fetch/body.js +45 -61
  58. package/lib/web/fetch/data-url.js +12 -160
  59. package/lib/web/fetch/formdata-parser.js +204 -127
  60. package/lib/web/fetch/index.js +21 -19
  61. package/lib/web/fetch/request.js +6 -0
  62. package/lib/web/fetch/response.js +4 -7
  63. package/lib/web/fetch/util.js +10 -79
  64. package/lib/web/infra/index.js +229 -0
  65. package/lib/web/subresource-integrity/subresource-integrity.js +6 -5
  66. package/lib/web/webidl/index.js +207 -44
  67. package/lib/web/websocket/connection.js +33 -22
  68. package/lib/web/websocket/events.js +1 -1
  69. package/lib/web/websocket/frame.js +9 -15
  70. package/lib/web/websocket/stream/websocketerror.js +22 -1
  71. package/lib/web/websocket/stream/websocketstream.js +17 -8
  72. package/lib/web/websocket/util.js +2 -1
  73. package/lib/web/websocket/websocket.js +32 -42
  74. package/package.json +9 -7
  75. package/types/agent.d.ts +2 -1
  76. package/types/api.d.ts +2 -2
  77. package/types/balanced-pool.d.ts +2 -1
  78. package/types/cache-interceptor.d.ts +1 -0
  79. package/types/client.d.ts +1 -1
  80. package/types/connector.d.ts +2 -2
  81. package/types/diagnostics-channel.d.ts +2 -2
  82. package/types/dispatcher.d.ts +12 -12
  83. package/types/errors.d.ts +5 -15
  84. package/types/fetch.d.ts +4 -4
  85. package/types/formdata.d.ts +1 -1
  86. package/types/h2c-client.d.ts +1 -1
  87. package/types/index.d.ts +9 -1
  88. package/types/interceptors.d.ts +36 -2
  89. package/types/pool.d.ts +1 -1
  90. package/types/readable.d.ts +2 -2
  91. package/types/round-robin-pool.d.ts +41 -0
  92. package/types/webidl.d.ts +82 -21
  93. package/types/websocket.d.ts +9 -9
@@ -0,0 +1,124 @@
1
+ 'use strict'
2
+
3
+ /** @typedef {`node:${string}`} NodeModuleName */
4
+
5
+ /** @type {Record<NodeModuleName, () => any>} */
6
+ const lazyLoaders = {
7
+ __proto__: null,
8
+ 'node:crypto': () => require('node:crypto'),
9
+ 'node:sqlite': () => require('node:sqlite'),
10
+ 'node:worker_threads': () => require('node:worker_threads'),
11
+ 'node:zlib': () => require('node:zlib')
12
+ }
13
+
14
+ /**
15
+ * @param {NodeModuleName} moduleName
16
+ * @returns {boolean}
17
+ */
18
+ function detectRuntimeFeatureByNodeModule (moduleName) {
19
+ try {
20
+ lazyLoaders[moduleName]()
21
+ return true
22
+ } catch (err) {
23
+ if (err.code !== 'ERR_UNKNOWN_BUILTIN_MODULE') {
24
+ throw err
25
+ }
26
+ return false
27
+ }
28
+ }
29
+
30
+ /**
31
+ * @param {NodeModuleName} moduleName
32
+ * @param {string} property
33
+ * @returns {boolean}
34
+ */
35
+ function detectRuntimeFeatureByExportedProperty (moduleName, property) {
36
+ const module = lazyLoaders[moduleName]()
37
+ return typeof module[property] !== 'undefined'
38
+ }
39
+
40
+ const runtimeFeaturesByExportedProperty = /** @type {const} */ (['markAsUncloneable', 'zstd'])
41
+
42
+ /** @type {Record<RuntimeFeatureByExportedProperty, [NodeModuleName, string]>} */
43
+ const exportedPropertyLookup = {
44
+ markAsUncloneable: ['node:worker_threads', 'markAsUncloneable'],
45
+ zstd: ['node:zlib', 'createZstdDecompress']
46
+ }
47
+
48
+ /** @typedef {typeof runtimeFeaturesByExportedProperty[number]} RuntimeFeatureByExportedProperty */
49
+
50
+ const runtimeFeaturesAsNodeModule = /** @type {const} */ (['crypto', 'sqlite'])
51
+ /** @typedef {typeof runtimeFeaturesAsNodeModule[number]} RuntimeFeatureByNodeModule */
52
+
53
+ const features = /** @type {const} */ ([
54
+ ...runtimeFeaturesAsNodeModule,
55
+ ...runtimeFeaturesByExportedProperty
56
+ ])
57
+
58
+ /** @typedef {typeof features[number]} Feature */
59
+
60
+ /**
61
+ * @param {Feature} feature
62
+ * @returns {boolean}
63
+ */
64
+ function detectRuntimeFeature (feature) {
65
+ if (runtimeFeaturesAsNodeModule.includes(/** @type {RuntimeFeatureByNodeModule} */ (feature))) {
66
+ return detectRuntimeFeatureByNodeModule(`node:${feature}`)
67
+ } else if (runtimeFeaturesByExportedProperty.includes(/** @type {RuntimeFeatureByExportedProperty} */ (feature))) {
68
+ const [moduleName, property] = exportedPropertyLookup[feature]
69
+ return detectRuntimeFeatureByExportedProperty(moduleName, property)
70
+ }
71
+ throw new TypeError(`unknown feature: ${feature}`)
72
+ }
73
+
74
+ /**
75
+ * @class
76
+ * @name RuntimeFeatures
77
+ */
78
+ class RuntimeFeatures {
79
+ /** @type {Map<Feature, boolean>} */
80
+ #map = new Map()
81
+
82
+ /**
83
+ * Clears all cached feature detections.
84
+ */
85
+ clear () {
86
+ this.#map.clear()
87
+ }
88
+
89
+ /**
90
+ * @param {Feature} feature
91
+ * @returns {boolean}
92
+ */
93
+ has (feature) {
94
+ return (
95
+ this.#map.get(feature) ?? this.#detectRuntimeFeature(feature)
96
+ )
97
+ }
98
+
99
+ /**
100
+ * @param {Feature} feature
101
+ * @param {boolean} value
102
+ */
103
+ set (feature, value) {
104
+ if (features.includes(feature) === false) {
105
+ throw new TypeError(`unknown feature: ${feature}`)
106
+ }
107
+ this.#map.set(feature, value)
108
+ }
109
+
110
+ /**
111
+ * @param {Feature} feature
112
+ * @returns {boolean}
113
+ */
114
+ #detectRuntimeFeature (feature) {
115
+ const result = detectRuntimeFeature(feature)
116
+ this.#map.set(feature, result)
117
+ return result
118
+ }
119
+ }
120
+
121
+ const instance = new RuntimeFeatures()
122
+
123
+ module.exports.runtimeFeatures = instance
124
+ module.exports.default = instance
@@ -186,7 +186,7 @@ webidl.converters.Cookie = webidl.dictionaryConverter([
186
186
  {
187
187
  converter: webidl.sequenceConverter(webidl.converters.DOMString),
188
188
  key: 'unparsed',
189
- defaultValue: () => new Array(0)
189
+ defaultValue: () => []
190
190
  }
191
191
  ])
192
192
 
@@ -1,8 +1,8 @@
1
1
  'use strict'
2
2
 
3
+ const { collectASequenceOfCodePointsFast } = require('../infra')
3
4
  const { maxNameValuePairSize, maxAttributeValueSize } = require('./constants')
4
5
  const { isCTLExcludingHtab } = require('./util')
5
- const { collectASequenceOfCodePointsFast } = require('../fetch/data-url')
6
6
  const assert = require('node:assert')
7
7
  const { unescape: qsUnescape } = require('node:querystring')
8
8
 
@@ -236,7 +236,7 @@ class EventSourceStream extends Transform {
236
236
  this.buffer = this.buffer.subarray(this.pos + 1)
237
237
  this.pos = 0
238
238
  if (
239
- this.event.data !== undefined || this.event.event || this.event.id || this.event.retry) {
239
+ this.event.data !== undefined || this.event.event || this.event.id !== undefined || this.event.retry) {
240
240
  this.processEvent(this.event)
241
241
  }
242
242
  this.clearEvent()
@@ -367,7 +367,7 @@ class EventSourceStream extends Transform {
367
367
  this.state.reconnectionTime = parseInt(event.retry, 10)
368
368
  }
369
369
 
370
- if (event.id && isValidLastEventId(event.id)) {
370
+ if (event.id !== undefined && isValidLastEventId(event.id)) {
371
371
  this.state.lastEventId = event.id
372
372
  }
373
373
 
@@ -8,7 +8,6 @@ const { EventSourceStream } = require('./eventsource-stream')
8
8
  const { parseMIMEType } = require('../fetch/data-url')
9
9
  const { createFastMessageEvent } = require('../websocket/events')
10
10
  const { isNetworkError } = require('../fetch/response')
11
- const { delay } = require('./util')
12
11
  const { kEnumerableProperty } = require('../../core/util')
13
12
  const { environmentSettingsObject } = require('../fetch/util')
14
13
 
@@ -318,9 +317,9 @@ class EventSource extends EventTarget {
318
317
 
319
318
  /**
320
319
  * @see https://html.spec.whatwg.org/multipage/server-sent-events.html#sse-processing-model
321
- * @returns {Promise<void>}
320
+ * @returns {void}
322
321
  */
323
- async #reconnect () {
322
+ #reconnect () {
324
323
  // When a user agent is to reestablish the connection, the user agent must
325
324
  // run the following steps. These steps are run in parallel, not as part of
326
325
  // a task. (The tasks that it queues, of course, are run like normal tasks
@@ -338,27 +337,27 @@ class EventSource extends EventTarget {
338
337
  this.dispatchEvent(new Event('error'))
339
338
 
340
339
  // 2. Wait a delay equal to the reconnection time of the event source.
341
- await delay(this.#state.reconnectionTime)
342
-
343
- // 5. Queue a task to run the following steps:
344
-
345
- // 1. If the EventSource object's readyState attribute is not set to
346
- // CONNECTING, then return.
347
- if (this.#readyState !== CONNECTING) return
348
-
349
- // 2. Let request be the EventSource object's request.
350
- // 3. If the EventSource object's last event ID string is not the empty
351
- // string, then:
352
- // 1. Let lastEventIDValue be the EventSource object's last event ID
353
- // string, encoded as UTF-8.
354
- // 2. Set (`Last-Event-ID`, lastEventIDValue) in request's header
355
- // list.
356
- if (this.#state.lastEventId.length) {
357
- this.#request.headersList.set('last-event-id', this.#state.lastEventId, true)
358
- }
340
+ setTimeout(() => {
341
+ // 5. Queue a task to run the following steps:
342
+
343
+ // 1. If the EventSource object's readyState attribute is not set to
344
+ // CONNECTING, then return.
345
+ if (this.#readyState !== CONNECTING) return
346
+
347
+ // 2. Let request be the EventSource object's request.
348
+ // 3. If the EventSource object's last event ID string is not the empty
349
+ // string, then:
350
+ // 1. Let lastEventIDValue be the EventSource object's last event ID
351
+ // string, encoded as UTF-8.
352
+ // 2. Set (`Last-Event-ID`, lastEventIDValue) in request's header
353
+ // list.
354
+ if (this.#state.lastEventId.length) {
355
+ this.#request.headersList.set('last-event-id', this.#state.lastEventId, true)
356
+ }
359
357
 
360
- // 4. Fetch request and process the response obtained in this fashion, if any, as described earlier in this section.
361
- this.#connect()
358
+ // 4. Fetch request and process the response obtained in this fashion, if any, as described earlier in this section.
359
+ this.#connect()
360
+ }, this.#state.reconnectionTime)?.unref()
362
361
  }
363
362
 
364
363
  /**
@@ -383,9 +382,11 @@ class EventSource extends EventTarget {
383
382
  this.removeEventListener('open', this.#events.open)
384
383
  }
385
384
 
386
- if (typeof fn === 'function') {
385
+ const listener = webidl.converters.EventHandlerNonNull(fn)
386
+
387
+ if (listener !== null) {
388
+ this.addEventListener('open', listener)
387
389
  this.#events.open = fn
388
- this.addEventListener('open', fn)
389
390
  } else {
390
391
  this.#events.open = null
391
392
  }
@@ -400,9 +401,11 @@ class EventSource extends EventTarget {
400
401
  this.removeEventListener('message', this.#events.message)
401
402
  }
402
403
 
403
- if (typeof fn === 'function') {
404
+ const listener = webidl.converters.EventHandlerNonNull(fn)
405
+
406
+ if (listener !== null) {
407
+ this.addEventListener('message', listener)
404
408
  this.#events.message = fn
405
- this.addEventListener('message', fn)
406
409
  } else {
407
410
  this.#events.message = null
408
411
  }
@@ -417,9 +420,11 @@ class EventSource extends EventTarget {
417
420
  this.removeEventListener('error', this.#events.error)
418
421
  }
419
422
 
420
- if (typeof fn === 'function') {
423
+ const listener = webidl.converters.EventHandlerNonNull(fn)
424
+
425
+ if (listener !== null) {
426
+ this.addEventListener('error', listener)
421
427
  this.#events.error = fn
422
- this.addEventListener('error', fn)
423
428
  } else {
424
429
  this.#events.error = null
425
430
  }
@@ -23,15 +23,7 @@ function isASCIINumber (value) {
23
23
  return true
24
24
  }
25
25
 
26
- // https://github.com/nodejs/undici/issues/2664
27
- function delay (ms) {
28
- return new Promise((resolve) => {
29
- setTimeout(resolve, ms)
30
- })
31
- }
32
-
33
26
  module.exports = {
34
27
  isValidLastEventId,
35
- isASCIINumber,
36
- delay
28
+ isASCIINumber
37
29
  }
@@ -5,8 +5,7 @@ const {
5
5
  ReadableStreamFrom,
6
6
  readableStreamClose,
7
7
  fullyReadBody,
8
- extractMimeType,
9
- utf8DecodeBytes
8
+ extractMimeType
10
9
  } = require('./util')
11
10
  const { FormData, setFormDataState } = require('./formdata')
12
11
  const { webidl } = require('../webidl')
@@ -16,15 +15,13 @@ const { isArrayBuffer } = require('node:util/types')
16
15
  const { serializeAMimeType } = require('./data-url')
17
16
  const { multipartFormDataParser } = require('./formdata-parser')
18
17
  const { createDeferredPromise } = require('../../util/promise')
18
+ const { parseJSONFromBytes } = require('../infra')
19
+ const { utf8DecodeBytes } = require('../../encoding')
20
+ const { runtimeFeatures } = require('../../util/runtime-features.js')
19
21
 
20
- let random
21
-
22
- try {
23
- const crypto = require('node:crypto')
24
- random = (max) => crypto.randomInt(0, max)
25
- } catch {
26
- random = (max) => Math.floor(Math.random() * max)
27
- }
22
+ const random = runtimeFeatures.has('crypto')
23
+ ? require('node:crypto').randomInt
24
+ : (max) => Math.floor(Math.random() * max)
28
25
 
29
26
  const textEncoder = new TextEncoder()
30
27
  function noop () {}
@@ -60,7 +57,7 @@ function extractBody (object, keepalive = false) {
60
57
  // 4. Otherwise, set stream to a new ReadableStream object, and set
61
58
  // up stream with byte reading support.
62
59
  stream = new ReadableStream({
63
- async pull (controller) {
60
+ pull (controller) {
64
61
  const buffer = typeof source === 'string' ? textEncoder.encode(source) : source
65
62
 
66
63
  if (buffer.byteLength) {
@@ -110,16 +107,10 @@ function extractBody (object, keepalive = false) {
110
107
 
111
108
  // Set type to `application/x-www-form-urlencoded;charset=UTF-8`.
112
109
  type = 'application/x-www-form-urlencoded;charset=UTF-8'
113
- } else if (isArrayBuffer(object)) {
114
- // BufferSource/ArrayBuffer
115
-
116
- // Set source to a copy of the bytes held by object.
117
- source = new Uint8Array(object.slice())
118
- } else if (ArrayBuffer.isView(object)) {
119
- // BufferSource/ArrayBufferView
120
-
121
- // Set source to a copy of the bytes held by object.
122
- source = new Uint8Array(object.buffer.slice(object.byteOffset, object.byteOffset + object.byteLength))
110
+ } else if (webidl.is.BufferSource(object)) {
111
+ source = isArrayBuffer(object)
112
+ ? new Uint8Array(object.slice())
113
+ : new Uint8Array(object.buffer.slice(object.byteOffset, object.byteOffset + object.byteLength))
123
114
  } else if (webidl.is.FormData(object)) {
124
115
  const boundary = `----formdata-undici-0${`${random(1e11)}`.padStart(11, '0')}`
125
116
  const prefix = `--${boundary}\r\nContent-Disposition: form-data`
@@ -231,32 +222,33 @@ function extractBody (object, keepalive = false) {
231
222
  // Run action.
232
223
  let iterator
233
224
  stream = new ReadableStream({
234
- async start () {
225
+ start () {
235
226
  iterator = action(object)[Symbol.asyncIterator]()
236
227
  },
237
- async pull (controller) {
238
- const { value, done } = await iterator.next()
239
- if (done) {
240
- // When running action is done, close stream.
241
- queueMicrotask(() => {
242
- controller.close()
243
- controller.byobRequest?.respond(0)
244
- })
245
- } else {
246
- // Whenever one or more bytes are available and stream is not errored,
247
- // enqueue a Uint8Array wrapping an ArrayBuffer containing the available
248
- // bytes into stream.
249
- if (!isErrored(stream)) {
250
- const buffer = new Uint8Array(value)
251
- if (buffer.byteLength) {
252
- controller.enqueue(buffer)
228
+ pull (controller) {
229
+ return iterator.next().then(({ value, done }) => {
230
+ if (done) {
231
+ // When running action is done, close stream.
232
+ queueMicrotask(() => {
233
+ controller.close()
234
+ controller.byobRequest?.respond(0)
235
+ })
236
+ } else {
237
+ // Whenever one or more bytes are available and stream is not errored,
238
+ // enqueue a Uint8Array wrapping an ArrayBuffer containing the available
239
+ // bytes into stream.
240
+ if (!isErrored(stream)) {
241
+ const buffer = new Uint8Array(value)
242
+ if (buffer.byteLength) {
243
+ controller.enqueue(buffer)
244
+ }
253
245
  }
254
246
  }
255
- }
256
- return controller.desiredSize > 0
247
+ return controller.desiredSize > 0
248
+ })
257
249
  },
258
- async cancel (reason) {
259
- await iterator.return()
250
+ cancel (reason) {
251
+ return iterator.return()
260
252
  },
261
253
  type: 'bytes'
262
254
  })
@@ -320,12 +312,6 @@ function cloneBody (body) {
320
312
  }
321
313
  }
322
314
 
323
- function throwIfAborted (state) {
324
- if (state.aborted) {
325
- throw new DOMException('The operation was aborted.', 'AbortError')
326
- }
327
- }
328
-
329
315
  function bodyMixinMethods (instance, getInternalState) {
330
316
  const methods = {
331
317
  blob () {
@@ -443,24 +429,30 @@ function mixinBody (prototype, getInternalState) {
443
429
  * @param {any} instance
444
430
  * @param {(target: any) => any} getInternalState
445
431
  */
446
- async function consumeBody (object, convertBytesToJSValue, instance, getInternalState) {
447
- webidl.brandCheck(object, instance)
432
+ function consumeBody (object, convertBytesToJSValue, instance, getInternalState) {
433
+ try {
434
+ webidl.brandCheck(object, instance)
435
+ } catch (e) {
436
+ return Promise.reject(e)
437
+ }
448
438
 
449
439
  const state = getInternalState(object)
450
440
 
451
441
  // 1. If object is unusable, then return a promise rejected
452
442
  // with a TypeError.
453
443
  if (bodyUnusable(state)) {
454
- throw new TypeError('Body is unusable: Body has already been read')
444
+ return Promise.reject(new TypeError('Body is unusable: Body has already been read'))
455
445
  }
456
446
 
457
- throwIfAborted(state)
447
+ if (state.aborted) {
448
+ return Promise.reject(new DOMException('The operation was aborted.', 'AbortError'))
449
+ }
458
450
 
459
451
  // 2. Let promise be a new promise.
460
452
  const promise = createDeferredPromise()
461
453
 
462
454
  // 3. Let errorSteps given error be to reject promise with error.
463
- const errorSteps = (error) => promise.reject(error)
455
+ const errorSteps = promise.reject
464
456
 
465
457
  // 4. Let successSteps given a byte sequence data be to resolve
466
458
  // promise with the result of running convertBytesToJSValue
@@ -502,14 +494,6 @@ function bodyUnusable (object) {
502
494
  return body != null && (body.stream.locked || util.isDisturbed(body.stream))
503
495
  }
504
496
 
505
- /**
506
- * @see https://infra.spec.whatwg.org/#parse-json-bytes-to-a-javascript-value
507
- * @param {Uint8Array} bytes
508
- */
509
- function parseJSONFromBytes (bytes) {
510
- return JSON.parse(utf8DecodeBytes(bytes))
511
- }
512
-
513
497
  /**
514
498
  * @see https://fetch.spec.whatwg.org/#concept-body-mime-type
515
499
  * @param {any} requestOrResponse internal state