undici 7.15.0 → 7.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/README.md +48 -2
  2. package/docs/docs/api/Agent.md +1 -0
  3. package/docs/docs/api/Client.md +1 -0
  4. package/docs/docs/api/DiagnosticsChannel.md +57 -0
  5. package/docs/docs/api/Dispatcher.md +86 -0
  6. package/docs/docs/api/Errors.md +0 -1
  7. package/docs/docs/api/RoundRobinPool.md +145 -0
  8. package/docs/docs/api/WebSocket.md +21 -0
  9. package/docs/docs/best-practices/crawling.md +58 -0
  10. package/index-fetch.js +2 -2
  11. package/index.js +8 -9
  12. package/lib/api/api-request.js +22 -8
  13. package/lib/api/api-upgrade.js +2 -1
  14. package/lib/api/readable.js +7 -5
  15. package/lib/core/connect.js +4 -1
  16. package/lib/core/diagnostics.js +28 -1
  17. package/lib/core/errors.js +217 -13
  18. package/lib/core/request.js +5 -1
  19. package/lib/core/symbols.js +3 -0
  20. package/lib/core/util.js +61 -41
  21. package/lib/dispatcher/agent.js +19 -7
  22. package/lib/dispatcher/balanced-pool.js +10 -0
  23. package/lib/dispatcher/client-h1.js +18 -23
  24. package/lib/dispatcher/client-h2.js +166 -26
  25. package/lib/dispatcher/client.js +64 -59
  26. package/lib/dispatcher/dispatcher-base.js +20 -16
  27. package/lib/dispatcher/env-http-proxy-agent.js +12 -16
  28. package/lib/dispatcher/fixed-queue.js +15 -39
  29. package/lib/dispatcher/h2c-client.js +7 -78
  30. package/lib/dispatcher/pool-base.js +60 -43
  31. package/lib/dispatcher/pool.js +2 -2
  32. package/lib/dispatcher/proxy-agent.js +27 -11
  33. package/lib/dispatcher/round-robin-pool.js +137 -0
  34. package/lib/encoding/index.js +33 -0
  35. package/lib/global.js +19 -1
  36. package/lib/handler/cache-handler.js +84 -27
  37. package/lib/handler/deduplication-handler.js +216 -0
  38. package/lib/handler/retry-handler.js +0 -2
  39. package/lib/interceptor/cache.js +94 -15
  40. package/lib/interceptor/decompress.js +2 -1
  41. package/lib/interceptor/deduplicate.js +109 -0
  42. package/lib/interceptor/dns.js +55 -13
  43. package/lib/mock/mock-agent.js +4 -4
  44. package/lib/mock/mock-errors.js +10 -0
  45. package/lib/mock/mock-utils.js +13 -12
  46. package/lib/mock/snapshot-agent.js +11 -5
  47. package/lib/mock/snapshot-recorder.js +12 -4
  48. package/lib/mock/snapshot-utils.js +4 -4
  49. package/lib/util/cache.js +29 -1
  50. package/lib/util/date.js +534 -140
  51. package/lib/util/runtime-features.js +124 -0
  52. package/lib/web/cookies/index.js +1 -1
  53. package/lib/web/cookies/parse.js +1 -1
  54. package/lib/web/eventsource/eventsource-stream.js +2 -2
  55. package/lib/web/eventsource/eventsource.js +34 -29
  56. package/lib/web/eventsource/util.js +1 -9
  57. package/lib/web/fetch/body.js +45 -61
  58. package/lib/web/fetch/data-url.js +12 -160
  59. package/lib/web/fetch/formdata-parser.js +204 -127
  60. package/lib/web/fetch/index.js +21 -19
  61. package/lib/web/fetch/request.js +6 -0
  62. package/lib/web/fetch/response.js +4 -7
  63. package/lib/web/fetch/util.js +10 -79
  64. package/lib/web/infra/index.js +229 -0
  65. package/lib/web/subresource-integrity/subresource-integrity.js +6 -5
  66. package/lib/web/webidl/index.js +207 -44
  67. package/lib/web/websocket/connection.js +33 -22
  68. package/lib/web/websocket/events.js +1 -1
  69. package/lib/web/websocket/frame.js +9 -15
  70. package/lib/web/websocket/stream/websocketerror.js +22 -1
  71. package/lib/web/websocket/stream/websocketstream.js +17 -8
  72. package/lib/web/websocket/util.js +2 -1
  73. package/lib/web/websocket/websocket.js +32 -42
  74. package/package.json +9 -7
  75. package/types/agent.d.ts +2 -1
  76. package/types/api.d.ts +2 -2
  77. package/types/balanced-pool.d.ts +2 -1
  78. package/types/cache-interceptor.d.ts +1 -0
  79. package/types/client.d.ts +1 -1
  80. package/types/connector.d.ts +2 -2
  81. package/types/diagnostics-channel.d.ts +2 -2
  82. package/types/dispatcher.d.ts +12 -12
  83. package/types/errors.d.ts +5 -15
  84. package/types/fetch.d.ts +4 -4
  85. package/types/formdata.d.ts +1 -1
  86. package/types/h2c-client.d.ts +1 -1
  87. package/types/index.d.ts +9 -1
  88. package/types/interceptors.d.ts +36 -2
  89. package/types/pool.d.ts +1 -1
  90. package/types/readable.d.ts +2 -2
  91. package/types/round-robin-pool.d.ts +41 -0
  92. package/types/webidl.d.ts +82 -21
  93. package/types/websocket.d.ts +9 -9
@@ -107,7 +107,8 @@ class Client extends DispatcherBase {
107
107
  autoSelectFamilyAttemptTimeout,
108
108
  // h2
109
109
  maxConcurrentStreams,
110
- allowH2
110
+ allowH2,
111
+ useH2c
111
112
  } = {}) {
112
113
  if (keepAlive !== undefined) {
113
114
  throw new InvalidArgumentError('unsupported keepAlive, use pipelining=0 instead')
@@ -199,6 +200,10 @@ class Client extends DispatcherBase {
199
200
  throw new InvalidArgumentError('maxConcurrentStreams must be a positive integer, greater than 0')
200
201
  }
201
202
 
203
+ if (useH2c != null && typeof useH2c !== 'boolean') {
204
+ throw new InvalidArgumentError('useH2c must be a valid boolean value')
205
+ }
206
+
202
207
  super()
203
208
 
204
209
  if (typeof connect !== 'function') {
@@ -206,6 +211,7 @@ class Client extends DispatcherBase {
206
211
  ...tls,
207
212
  maxCachedSessions,
208
213
  allowH2,
214
+ useH2c,
209
215
  socketPath,
210
216
  timeout: connectTimeout,
211
217
  ...(typeof autoSelectFamily === 'boolean' ? { autoSelectFamily, autoSelectFamilyAttemptTimeout } : undefined),
@@ -289,15 +295,13 @@ class Client extends DispatcherBase {
289
295
  )
290
296
  }
291
297
 
292
- /* istanbul ignore: only used for test */
293
298
  [kConnect] (cb) {
294
299
  connect(this)
295
300
  this.once('connect', cb)
296
301
  }
297
302
 
298
303
  [kDispatch] (opts, handler) {
299
- const origin = opts.origin || this[kUrl].origin
300
- const request = new Request(origin, opts, handler)
304
+ const request = new Request(this[kUrl].origin, opts, handler)
301
305
 
302
306
  this[kQueue].push(request)
303
307
  if (this[kResuming]) {
@@ -317,7 +321,7 @@ class Client extends DispatcherBase {
317
321
  return this[kNeedDrain] < 2
318
322
  }
319
323
 
320
- async [kClose] () {
324
+ [kClose] () {
321
325
  // TODO: for H2 we need to gracefully flush the remaining enqueued
322
326
  // request and close each stream.
323
327
  return new Promise((resolve) => {
@@ -329,7 +333,7 @@ class Client extends DispatcherBase {
329
333
  })
330
334
  }
331
335
 
332
- async [kDestroy] (err) {
336
+ [kDestroy] (err) {
333
337
  return new Promise((resolve) => {
334
338
  const requests = this[kQueue].splice(this[kPendingIdx])
335
339
  for (let i = 0; i < requests.length; i++) {
@@ -381,9 +385,9 @@ function onError (client, err) {
381
385
 
382
386
  /**
383
387
  * @param {Client} client
384
- * @returns
388
+ * @returns {void}
385
389
  */
386
- async function connect (client) {
390
+ function connect (client) {
387
391
  assert(!client[kConnecting])
388
392
  assert(!client[kHTTPContext])
389
393
 
@@ -417,26 +421,23 @@ async function connect (client) {
417
421
  })
418
422
  }
419
423
 
420
- try {
421
- const socket = await new Promise((resolve, reject) => {
422
- client[kConnector]({
423
- host,
424
- hostname,
425
- protocol,
426
- port,
427
- servername: client[kServerName],
428
- localAddress: client[kLocalAddress]
429
- }, (err, socket) => {
430
- if (err) {
431
- reject(err)
432
- } else {
433
- resolve(socket)
434
- }
435
- })
436
- })
424
+ client[kConnector]({
425
+ host,
426
+ hostname,
427
+ protocol,
428
+ port,
429
+ servername: client[kServerName],
430
+ localAddress: client[kLocalAddress]
431
+ }, (err, socket) => {
432
+ if (err) {
433
+ handleConnectError(client, err, { host, hostname, protocol, port })
434
+ client[kResume]()
435
+ return
436
+ }
437
437
 
438
438
  if (client.destroyed) {
439
439
  util.destroy(socket.on('error', noop), new ClientDestroyedError())
440
+ client[kResume]()
440
441
  return
441
442
  }
442
443
 
@@ -444,11 +445,13 @@ async function connect (client) {
444
445
 
445
446
  try {
446
447
  client[kHTTPContext] = socket.alpnProtocol === 'h2'
447
- ? await connectH2(client, socket)
448
- : await connectH1(client, socket)
448
+ ? connectH2(client, socket)
449
+ : connectH1(client, socket)
449
450
  } catch (err) {
450
451
  socket.destroy().on('error', noop)
451
- throw err
452
+ handleConnectError(client, err, { host, hostname, protocol, port })
453
+ client[kResume]()
454
+ return
452
455
  }
453
456
 
454
457
  client[kConnecting] = false
@@ -473,44 +476,46 @@ async function connect (client) {
473
476
  socket
474
477
  })
475
478
  }
479
+
476
480
  client.emit('connect', client[kUrl], [client])
477
- } catch (err) {
478
- if (client.destroyed) {
479
- return
480
- }
481
+ client[kResume]()
482
+ })
483
+ }
481
484
 
482
- client[kConnecting] = false
485
+ function handleConnectError (client, err, { host, hostname, protocol, port }) {
486
+ if (client.destroyed) {
487
+ return
488
+ }
483
489
 
484
- if (channels.connectError.hasSubscribers) {
485
- channels.connectError.publish({
486
- connectParams: {
487
- host,
488
- hostname,
489
- protocol,
490
- port,
491
- version: client[kHTTPContext]?.version,
492
- servername: client[kServerName],
493
- localAddress: client[kLocalAddress]
494
- },
495
- connector: client[kConnector],
496
- error: err
497
- })
498
- }
490
+ client[kConnecting] = false
499
491
 
500
- if (err.code === 'ERR_TLS_CERT_ALTNAME_INVALID') {
501
- assert(client[kRunning] === 0)
502
- while (client[kPending] > 0 && client[kQueue][client[kPendingIdx]].servername === client[kServerName]) {
503
- const request = client[kQueue][client[kPendingIdx]++]
504
- util.errorRequest(client, request, err)
505
- }
506
- } else {
507
- onError(client, err)
508
- }
492
+ if (channels.connectError.hasSubscribers) {
493
+ channels.connectError.publish({
494
+ connectParams: {
495
+ host,
496
+ hostname,
497
+ protocol,
498
+ port,
499
+ version: client[kHTTPContext]?.version,
500
+ servername: client[kServerName],
501
+ localAddress: client[kLocalAddress]
502
+ },
503
+ connector: client[kConnector],
504
+ error: err
505
+ })
506
+ }
509
507
 
510
- client.emit('connectionError', client[kUrl], [client], err)
508
+ if (err.code === 'ERR_TLS_CERT_ALTNAME_INVALID') {
509
+ assert(client[kRunning] === 0)
510
+ while (client[kPending] > 0 && client[kQueue][client[kPendingIdx]].servername === client[kServerName]) {
511
+ const request = client[kQueue][client[kPendingIdx]++]
512
+ util.errorRequest(client, request, err)
513
+ }
514
+ } else {
515
+ onError(client, err)
511
516
  }
512
517
 
513
- client[kResume]()
518
+ client.emit('connectionError', client[kUrl], [client], err)
514
519
  }
515
520
 
516
521
  function emitDrain (client) {
@@ -13,19 +13,24 @@ const kOnDestroyed = Symbol('onDestroyed')
13
13
  const kOnClosed = Symbol('onClosed')
14
14
 
15
15
  class DispatcherBase extends Dispatcher {
16
- constructor () {
17
- super()
16
+ /** @type {boolean} */
17
+ [kDestroyed] = false;
18
18
 
19
- this[kDestroyed] = false
20
- this[kOnDestroyed] = null
21
- this[kClosed] = false
22
- this[kOnClosed] = []
23
- }
19
+ /** @type {Array<Function|null} */
20
+ [kOnDestroyed] = null;
21
+
22
+ /** @type {boolean} */
23
+ [kClosed] = false;
24
+
25
+ /** @type {Array<Function>|null} */
26
+ [kOnClosed] = null
24
27
 
28
+ /** @returns {boolean} */
25
29
  get destroyed () {
26
30
  return this[kDestroyed]
27
31
  }
28
32
 
33
+ /** @returns {boolean} */
29
34
  get closed () {
30
35
  return this[kClosed]
31
36
  }
@@ -44,7 +49,8 @@ class DispatcherBase extends Dispatcher {
44
49
  }
45
50
 
46
51
  if (this[kDestroyed]) {
47
- queueMicrotask(() => callback(new ClientDestroyedError(), null))
52
+ const err = new ClientDestroyedError()
53
+ queueMicrotask(() => callback(err, null))
48
54
  return
49
55
  }
50
56
 
@@ -58,6 +64,7 @@ class DispatcherBase extends Dispatcher {
58
64
  }
59
65
 
60
66
  this[kClosed] = true
67
+ this[kOnClosed] ??= []
61
68
  this[kOnClosed].push(callback)
62
69
 
63
70
  const onClosed = () => {
@@ -71,9 +78,7 @@ class DispatcherBase extends Dispatcher {
71
78
  // Should not error.
72
79
  this[kClose]()
73
80
  .then(() => this.destroy())
74
- .then(() => {
75
- queueMicrotask(onClosed)
76
- })
81
+ .then(() => queueMicrotask(onClosed))
77
82
  }
78
83
 
79
84
  destroy (err, callback) {
@@ -85,7 +90,7 @@ class DispatcherBase extends Dispatcher {
85
90
  if (callback === undefined) {
86
91
  return new Promise((resolve, reject) => {
87
92
  this.destroy(err, (err, data) => {
88
- return err ? /* istanbul ignore next: should never error */ reject(err) : resolve(data)
93
+ return err ? reject(err) : resolve(data)
89
94
  })
90
95
  })
91
96
  }
@@ -108,7 +113,7 @@ class DispatcherBase extends Dispatcher {
108
113
  }
109
114
 
110
115
  this[kDestroyed] = true
111
- this[kOnDestroyed] = this[kOnDestroyed] || []
116
+ this[kOnDestroyed] ??= []
112
117
  this[kOnDestroyed].push(callback)
113
118
 
114
119
  const onDestroyed = () => {
@@ -120,9 +125,8 @@ class DispatcherBase extends Dispatcher {
120
125
  }
121
126
 
122
127
  // Should not error.
123
- this[kDestroy](err).then(() => {
124
- queueMicrotask(onDestroyed)
125
- })
128
+ this[kDestroy](err)
129
+ .then(() => queueMicrotask(onDestroyed))
126
130
  }
127
131
 
128
132
  dispatch (opts, handler) {
@@ -46,24 +46,20 @@ class EnvHttpProxyAgent extends DispatcherBase {
46
46
  return agent.dispatch(opts, handler)
47
47
  }
48
48
 
49
- async [kClose] () {
50
- await this[kNoProxyAgent].close()
51
- if (!this[kHttpProxyAgent][kClosed]) {
52
- await this[kHttpProxyAgent].close()
53
- }
54
- if (!this[kHttpsProxyAgent][kClosed]) {
55
- await this[kHttpsProxyAgent].close()
56
- }
49
+ [kClose] () {
50
+ return Promise.all([
51
+ this[kNoProxyAgent].close(),
52
+ !this[kHttpProxyAgent][kClosed] && this[kHttpProxyAgent].close(),
53
+ !this[kHttpsProxyAgent][kClosed] && this[kHttpsProxyAgent].close()
54
+ ])
57
55
  }
58
56
 
59
- async [kDestroy] (err) {
60
- await this[kNoProxyAgent].destroy(err)
61
- if (!this[kHttpProxyAgent][kDestroyed]) {
62
- await this[kHttpProxyAgent].destroy(err)
63
- }
64
- if (!this[kHttpsProxyAgent][kDestroyed]) {
65
- await this[kHttpsProxyAgent].destroy(err)
66
- }
57
+ [kDestroy] (err) {
58
+ return Promise.all([
59
+ this[kNoProxyAgent].destroy(err),
60
+ !this[kHttpProxyAgent][kDestroyed] && this[kHttpProxyAgent].destroy(err),
61
+ !this[kHttpsProxyAgent][kDestroyed] && this[kHttpsProxyAgent].destroy(err)
62
+ ])
67
63
  }
68
64
 
69
65
  #getProxyAgentForUrl (url) {
@@ -59,35 +59,21 @@ const kMask = kSize - 1
59
59
  * @template T
60
60
  */
61
61
  class FixedCircularBuffer {
62
- constructor () {
63
- /**
64
- * @type {number}
65
- */
66
- this.bottom = 0
67
- /**
68
- * @type {number}
69
- */
70
- this.top = 0
71
- /**
72
- * @type {Array<T|undefined>}
73
- */
74
- this.list = new Array(kSize).fill(undefined)
75
- /**
76
- * @type {T|null}
77
- */
78
- this.next = null
79
- }
62
+ /** @type {number} */
63
+ bottom = 0
64
+ /** @type {number} */
65
+ top = 0
66
+ /** @type {Array<T|undefined>} */
67
+ list = new Array(kSize).fill(undefined)
68
+ /** @type {T|null} */
69
+ next = null
80
70
 
81
- /**
82
- * @returns {boolean}
83
- */
71
+ /** @returns {boolean} */
84
72
  isEmpty () {
85
73
  return this.top === this.bottom
86
74
  }
87
75
 
88
- /**
89
- * @returns {boolean}
90
- */
76
+ /** @returns {boolean} */
91
77
  isFull () {
92
78
  return ((this.top + 1) & kMask) === this.bottom
93
79
  }
@@ -101,9 +87,7 @@ class FixedCircularBuffer {
101
87
  this.top = (this.top + 1) & kMask
102
88
  }
103
89
 
104
- /**
105
- * @returns {T|null}
106
- */
90
+ /** @returns {T|null} */
107
91
  shift () {
108
92
  const nextItem = this.list[this.bottom]
109
93
  if (nextItem === undefined) { return null }
@@ -118,22 +102,16 @@ class FixedCircularBuffer {
118
102
  */
119
103
  module.exports = class FixedQueue {
120
104
  constructor () {
121
- /**
122
- * @type {FixedCircularBuffer<T>}
123
- */
105
+ /** @type {FixedCircularBuffer<T>} */
124
106
  this.head = this.tail = new FixedCircularBuffer()
125
107
  }
126
108
 
127
- /**
128
- * @returns {boolean}
129
- */
109
+ /** @returns {boolean} */
130
110
  isEmpty () {
131
111
  return this.head.isEmpty()
132
112
  }
133
113
 
134
- /**
135
- * @param {T} data
136
- */
114
+ /** @param {T} data */
137
115
  push (data) {
138
116
  if (this.head.isFull()) {
139
117
  // Head is full: Creates a new queue, sets the old queue's `.next` to it,
@@ -143,9 +121,7 @@ module.exports = class FixedQueue {
143
121
  this.head.push(data)
144
122
  }
145
123
 
146
- /**
147
- * @returns {T|null}
148
- */
124
+ /** @returns {T|null} */
149
125
  shift () {
150
126
  const tail = this.tail
151
127
  const next = tail.shift()
@@ -1,19 +1,10 @@
1
1
  'use strict'
2
- const { connect } = require('node:net')
3
2
 
4
- const { kClose, kDestroy } = require('../core/symbols')
5
3
  const { InvalidArgumentError } = require('../core/errors')
6
- const util = require('../core/util')
7
-
8
4
  const Client = require('./client')
9
- const DispatcherBase = require('./dispatcher-base')
10
-
11
- class H2CClient extends DispatcherBase {
12
- #client = null
13
5
 
6
+ class H2CClient extends Client {
14
7
  constructor (origin, clientOpts) {
15
- super()
16
-
17
8
  if (typeof origin === 'string') {
18
9
  origin = new URL(origin)
19
10
  }
@@ -25,14 +16,14 @@ class H2CClient extends DispatcherBase {
25
16
  }
26
17
 
27
18
  const { connect, maxConcurrentStreams, pipelining, ...opts } =
28
- clientOpts ?? {}
19
+ clientOpts ?? {}
29
20
  let defaultMaxConcurrentStreams = 100
30
21
  let defaultPipelining = 100
31
22
 
32
23
  if (
33
24
  maxConcurrentStreams != null &&
34
- Number.isInteger(maxConcurrentStreams) &&
35
- maxConcurrentStreams > 0
25
+ Number.isInteger(maxConcurrentStreams) &&
26
+ maxConcurrentStreams > 0
36
27
  ) {
37
28
  defaultMaxConcurrentStreams = maxConcurrentStreams
38
29
  }
@@ -47,76 +38,14 @@ class H2CClient extends DispatcherBase {
47
38
  )
48
39
  }
49
40
 
50
- this.#client = new Client(origin, {
41
+ super(origin, {
51
42
  ...opts,
52
- connect: this.#buildConnector(connect),
53
43
  maxConcurrentStreams: defaultMaxConcurrentStreams,
54
44
  pipelining: defaultPipelining,
55
- allowH2: true
45
+ allowH2: true,
46
+ useH2c: true
56
47
  })
57
48
  }
58
-
59
- #buildConnector (connectOpts) {
60
- return (opts, callback) => {
61
- const timeout = connectOpts?.connectOpts ?? 10e3
62
- const { hostname, port, pathname } = opts
63
- const socket = connect({
64
- ...opts,
65
- host: hostname,
66
- port,
67
- pathname
68
- })
69
-
70
- // Set TCP keep alive options on the socket here instead of in connect() for the case of assigning the socket
71
- if (opts.keepAlive == null || opts.keepAlive) {
72
- const keepAliveInitialDelay =
73
- opts.keepAliveInitialDelay == null ? 60e3 : opts.keepAliveInitialDelay
74
- socket.setKeepAlive(true, keepAliveInitialDelay)
75
- }
76
-
77
- socket.alpnProtocol = 'h2'
78
-
79
- const clearConnectTimeout = util.setupConnectTimeout(
80
- new WeakRef(socket),
81
- { timeout, hostname, port }
82
- )
83
-
84
- socket
85
- .setNoDelay(true)
86
- .once('connect', function () {
87
- queueMicrotask(clearConnectTimeout)
88
-
89
- if (callback) {
90
- const cb = callback
91
- callback = null
92
- cb(null, this)
93
- }
94
- })
95
- .on('error', function (err) {
96
- queueMicrotask(clearConnectTimeout)
97
-
98
- if (callback) {
99
- const cb = callback
100
- callback = null
101
- cb(err)
102
- }
103
- })
104
-
105
- return socket
106
- }
107
- }
108
-
109
- dispatch (opts, handler) {
110
- return this.#client.dispatch(opts, handler)
111
- }
112
-
113
- async [kClose] () {
114
- await this.#client.close()
115
- }
116
-
117
- async [kDestroy] () {
118
- await this.#client.destroy()
119
- }
120
49
  }
121
50
 
122
51
  module.exports = H2CClient