undici 5.0.0 → 5.1.0

package/README.md

@@ -2,7 +2,7 @@
 
 [![Node CI](https://github.com/nodejs/undici/actions/workflows/nodejs.yml/badge.svg)](https://github.com/nodejs/undici/actions/workflows/nodejs.yml) [![js-standard-style](https://img.shields.io/badge/code%20style-standard-brightgreen.svg?style=flat)](http://standardjs.com/) [![npm version](https://badge.fury.io/js/undici.svg)](https://badge.fury.io/js/undici) [![codecov](https://codecov.io/gh/nodejs/undici/branch/main/graph/badge.svg?token=yZL6LtXkOA)](https://codecov.io/gh/nodejs/undici)
 
-A HTTP/1.1 client, written from scratch for Node.js.
+An HTTP/1.1 client, written from scratch for Node.js.
 
 > Undici means eleven in Italian. 1.1 -> 11 -> Eleven -> Undici.
 It is also a Stranger Things reference.
@@ -65,7 +65,15 @@ for await (const data of body) {
 console.log('trailers', trailers)
 ```
 
-Using [the body mixin from the Fetch Standard](https://fetch.spec.whatwg.org/#body-mixin).
+## Body Mixins
+
+The `body` mixins are the most common way to format the request/response body. Mixins include:
+
+- [`.formData()`](https://fetch.spec.whatwg.org/#dom-body-formdata)
+- [`.json()`](https://fetch.spec.whatwg.org/#dom-body-json)
+- [`.text()`](https://fetch.spec.whatwg.org/#dom-body-text)
+
+Example usage:
 
 ```js
 import { request } from 'undici'
@@ -83,6 +91,12 @@ console.log('data', await body.json())
 console.log('trailers', trailers)
 ```
 
+_Note: Once a mixin has been called then the body cannot be reused, thus calling additional mixins on `.body`, e.g. `.body.json(); .body.text()` will result in an error `TypeError: unusable` being thrown and returned through the `Promise` rejection._
+
+Should you need to access the `body` in plain-text after using a mixin, the best practice is to use the `.text()` mixin first and then manually parse the text to the desired format.
+
+For more information about their behavior, please reference the body mixin from the [Fetch Standard](https://fetch.spec.whatwg.org/#body-mixin).
+
 ## Common API Methods
 
 This section documents our most commonly used API methods. Additional APIs are documented in their own files within the [docs](./docs/) folder and are accessible via the navigation list on the left side of the docs site.
@@ -213,7 +227,7 @@ const data = {
 
 #### `response.body`
 
-Nodejs has two kinds of streams: [web streams](https://nodejs.org/dist/latest-v16.x/docs/api/webstreams.html) which follow the API of the WHATWG web standard found in browsers, and an older Node-specific [streams API](https://nodejs.org/api/stream.html). `response.body` returns a readable web stream. If you would prefer to work with a Node stream you can convert a web stream using `.fromWeb()`.
+Nodejs has two kinds of streams: [web streams](https://nodejs.org/dist/latest-v16.x/docs/api/webstreams.html), which follow the API of the WHATWG web standard found in browsers, and an older Node-specific [streams API](https://nodejs.org/api/stream.html). `response.body` returns a readable web stream. If you would prefer to work with a Node stream you can convert a web stream using `.fromWeb()`.
 
 ```js
     import {fetch} from 'undici';
@@ -228,7 +242,7 @@ Nodejs has two kinds of streams: [web streams](https://nodejs.org/dist/latest-v1
 
 #### Specification Compliance
 
-This section documents parts of the [Fetch Standard](https://fetch.spec.whatwg.org) which Undici does
+This section documents parts of the [Fetch Standard](https://fetch.spec.whatwg.org) that Undici does
 not support or does not fully implement.
 
 ##### Garbage Collection
@@ -239,7 +253,7 @@ The [Fetch Standard](https://fetch.spec.whatwg.org) allows users to skip consumi
 [garbage collection](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Memory_Management#garbage_collection) to release connection resources. Undici does not do the same. Therefore, it is important to always either consume or cancel the response body.
 
 Garbage collection in Node is less aggressive and deterministic
-(due to the lack of clear idle periods that browser have through the rendering refresh rate)
+(due to the lack of clear idle periods that browsers have through the rendering refresh rate)
 which means that leaving the release of connection resources to the garbage collector can lead
 to excessive connection usage, reduced performance (due to less connection re-use), and even
 stalls or deadlocks when running out of connections.
@@ -301,7 +315,7 @@ Returns: `Dispatcher`
 
 ## Specification Compliance
 
-This section documents parts of the HTTP/1.1 specification which Undici does
+This section documents parts of the HTTP/1.1 specification that Undici does
 not support or does not fully implement.
 
 ### Expect
@@ -334,7 +348,7 @@ aborted.
 
 ### Manual Redirect
 
-Since it is not possible to manually follow an HTTP redirect on server-side,
+Since it is not possible to manually follow an HTTP redirect on the server-side,
 Undici returns the actual response instead of an `opaqueredirect` filtered one
 when invoked with a `manual` redirect. This aligns `fetch()` with the other
 implementations in Deno and Cloudflare Workers.

package/docs/api/Dispatcher.md

@@ -193,7 +193,7 @@ Returns: `Boolean` - `false` if dispatcher is busy and further dispatch calls wo
 * **path** `string`
 * **method** `string`
 * **body** `string | Buffer | Uint8Array | stream.Readable | Iterable | AsyncIterable | null` (optional) - Default: `null`
-* **headers** `UndiciHeaders` (optional) - Default: `null`
+* **headers** `UndiciHeaders | string[]` (optional) - Default: `null`.
 * **idempotent** `boolean` (optional) - Default: `true` if `method` is `'HEAD'` or `'GET'` - Whether the requests can be safely retried or not. If `false` the request won't be sent until all preceding requests in the pipeline has completed.
 * **blocking** `boolean` (optional) - Default: `false` - Whether the response is expected to take a long time and would end up blocking the pipeline. When this is set to `true` further pipelining will be avoided on the same connection until headers have been received.
 * **upgrade** `string | null` (optional) - Default: `null` - Upgrade the request. Should be used to specify the kind of upgrade i.e. `'Websocket'`.

package/docs/api/MockAgent.md

@@ -445,3 +445,79 @@ mockAgent.disableNetConnect()
 await request('http://example.com')
 // Will throw
 ```
+
+### `MockAgent.pendingInterceptors()`
+
+This method returns any pending interceptors registered on a mock agent. A pending interceptor meets one of the following criteria:
+
+- Is registered with neither `.times(<number>)` nor `.persist()`, and has not been invoked;
+- Is persistent (i.e., registered with `.persist()`) and has not been invoked;
+- Is registered with `.times(<number>)` and has not been invoked `<number>` of times.
+
+Returns: `PendingInterceptor[]` (where `PendingInterceptor` is a `MockDispatch` with an additional `origin: string`)
+
+#### Example - List all pending inteceptors
+
+```js
+const agent = new MockAgent()
+agent.disableNetConnect()
+
+agent
+  .get('https://example.com')
+  .intercept({ method: 'GET', path: '/' })
+  .reply(200, '')
+
+const pendingInterceptors = agent.pendingInterceptors()
+// Returns [
+//   {
+//     timesInvoked: 0,
+//     times: 1,
+//     persist: false,
+//     consumed: false,
+//     pending: true,
+//     path: '/',
+//     method: 'GET',
+//     body: undefined,
+//     headers: undefined,
+//     data: {
+//       error: null,
+//       statusCode: 200,
+//       data: '',
+//       headers: {},
+//       trailers: {}
+//     },
+//     origin: 'https://example.com'
+//   }
+// ]
+```
+
+### `MockAgent.assertNoPendingInterceptors([options])`
+
+This method throws if the mock agent has any pending interceptors. A pending interceptor meets one of the following criteria:
+
+- Is registered with neither `.times(<number>)` nor `.persist()`, and has not been invoked;
+- Is persistent (i.e., registered with `.persist()`) and has not been invoked;
+- Is registered with `.times(<number>)` and has not been invoked `<number>` of times.
+
+#### Example - Check that there are no pending interceptors
+
+```js
+const agent = new MockAgent()
+agent.disableNetConnect()
+
+agent
+  .get('https://example.com')
+  .intercept({ method: 'GET', path: '/' })
+  .reply(200, '')
+
+agent.assertNoPendingInterceptors()
+// Throws an UndiciError with the following message:
+//
+// 1 interceptor is pending:
+//
+// ┌─────────┬────────┬───────────────────────┬──────┬─────────────┬────────────┬─────────────┬───────────┐
+// │ (index) │ Method │        Origin         │ Path │ Status code │ Persistent │ Invocations │ Remaining │
+// ├─────────┼────────┼───────────────────────┼──────┼─────────────┼────────────┼─────────────┼───────────┤
+// │    0    │ 'GET'  │ 'https://example.com' │ '/'  │     200     │    '❌'    │      0      │     1     │
+// └─────────┴────────┴───────────────────────┴──────┴─────────────┴────────────┴─────────────┴───────────┘
+```

package/docs/best-practices/mocking-request.md

@@ -5,17 +5,20 @@ Undici have its own mocking [utility](../api/MockAgent.md). It allow us to inter
 Example:
 
 ```js
-// index.mjs
+// bank.mjs
 import { request } from 'undici'
 
-export async function bankTransfer(recepient, ammount) {
-  const { body } = await request('http://localhost:3000/bank-transfer', 
+export async function bankTransfer(recepient, amount) {
+  const { body } = await request('http://localhost:3000/bank-transfer',
     {
       method: 'POST',
       headers: {
         'X-TOKEN-SECRET': 'SuperSecretToken',
       },
-      body: JSON.stringify({ recepient })
+      body: JSON.stringify({
+        recepient,
+        amount
+      })
     }
   )
   return await body.json()
@@ -28,7 +31,7 @@ And this is what the test file looks like:
 // index.test.mjs
 import { strict as assert } from 'assert'
 import { MockAgent, setGlobalDispatcher, } from 'undici'
-import { bankTransfer } from './undici.mjs'
+import { bankTransfer } from './bank.mjs'
 
 const mockAgent = new MockAgent();
 
@@ -46,7 +49,7 @@ mockPool.intercept({
   },
   body: JSON.stringify({
     recepient: '1234567890',
-    ammount: '100'
+    amount: '100'
   })
 }).reply(200, {
   message: 'transaction processed'
@@ -94,7 +97,7 @@ mockPool.intercept({
 
 const badRequest = await bankTransfer('1234567890', '100')
 // Will throw an error
-// MockNotMatchedError: Mock dispatch not matched for path '/bank-transfer': 
+// MockNotMatchedError: Mock dispatch not matched for path '/bank-transfer':
 // subsequent request to origin http://localhost:3000 was not allowed (net.connect disabled)
 ```
 

package/index.d.ts

@@ -16,6 +16,7 @@ import { request, pipeline, stream, connect, upgrade } from './types/api'
 export * from './types/fetch'
 export * from './types/file'
 export * from './types/formdata'
+export { Interceptable } from './types/mock-interceptor'
 
 export { Dispatcher, BalancedPool, Pool, Client, buildConnector, errors, Agent, request, stream, pipeline, connect, upgrade, setGlobalDispatcher, getGlobalDispatcher, MockClient, MockPool, MockAgent, mockErrors, ProxyAgent }
 export default Undici

package/lib/api/api-request.js

@@ -88,14 +88,16 @@ class RequestHandler extends AsyncResource {
     this.res = body
     const headers = this.responseHeaders === 'raw' ? util.parseRawHeaders(rawHeaders) : util.parseHeaders(rawHeaders)
 
-    this.runInAsyncScope(callback, null, null, {
-      statusCode,
-      headers,
-      trailers: this.trailers,
-      opaque,
-      body,
-      context
-    })
+    if (callback !== null) {
+      this.runInAsyncScope(callback, null, null, {
+        statusCode,
+        headers,
+        trailers: this.trailers,
+        opaque,
+        body,
+        context
+      })
+    }
   }
 
   onData (chunk) {

package/lib/core/request.js

@@ -11,6 +11,12 @@ const kHandler = Symbol('handler')
 
 const channels = {}
 
+let extractBody
+
+const nodeVersion = process.versions.node.split('.')
+const nodeMajor = Number(nodeVersion[0])
+const nodeMinor = Number(nodeVersion[1])
+
 try {
   const diagnosticsChannel = require('diagnostics_channel')
   channels.create = diagnosticsChannel.channel('undici:request:create')
@@ -79,7 +85,7 @@ class Request {
       this.body = body.byteLength ? body : null
     } else if (typeof body === 'string') {
       this.body = body.length ? Buffer.from(body) : null
-    } else if (util.isIterable(body) || util.isBlobLike(body)) {
+    } else if (util.isFormDataLike(body) || util.isIterable(body) || util.isBlobLike(body)) {
       this.body = body
     } else {
       throw new InvalidArgumentError('body must be a string, a Buffer, a Readable stream, an iterable, or an async iterable')
@@ -126,7 +132,22 @@ class Request {
       throw new InvalidArgumentError('headers must be an object or an array')
     }
 
-    if (util.isBlobLike(body) && this.contentType == null && body.type) {
+    if (util.isFormDataLike(this.body)) {
+      if (nodeMajor < 16 || (nodeMajor === 16 && nodeMinor < 5)) {
+        throw new InvalidArgumentError('Form-Data bodies are only supported in node v16.5 and newer.')
+      }
+
+      if (!extractBody) {
+        extractBody = require('../fetch/body.js').extractBody
+      }
+
+      const [bodyStream, contentType] = extractBody(body)
+      if (this.contentType == null) {
+        this.contentType = contentType
+        this.headers += `content-type: ${contentType}\r\n`
+      }
+      this.body = bodyStream.stream
+    } else if (util.isBlobLike(body) && this.contentType == null && body.type) {
       this.contentType = body.type
       this.headers += `content-type: ${body.type}\r\n`
     }

package/lib/core/util.js

@@ -324,6 +324,10 @@ function ReadableStreamFrom (iterable) {
   )
 }
 
+function isFormDataLike (chunk) {
+  return chunk && chunk.constructor && chunk.constructor.name === 'FormData'
+}
+
 const kEnumerableProperty = Object.create(null)
 kEnumerableProperty.enumerable = true
 
@@ -352,5 +356,6 @@ module.exports = {
   ReadableStreamFrom,
   isBuffer,
   validateHandler,
-  getSocketInfo
+  getSocketInfo,
+  isFormDataLike
 }

package/lib/fetch/body.js

@@ -71,7 +71,7 @@ function extractBody (object, keepalive = false) {
 
     // Set source to a copy of the bytes held by object.
     source = new Uint8Array(object)
-  } else if (object instanceof FormData) {
+  } else if (util.isFormDataLike(object)) {
     const boundary = '----formdata-undici-' + Math.random()
     const prefix = `--${boundary}\r\nContent-Disposition: form-data`
 
@@ -348,7 +348,7 @@ const properties = {
   bodyUsed: {
     enumerable: true,
     get () {
-      return this[kState].body && util.isDisturbed(this[kState].body.stream)
+      return !!this[kState].body && util.isDisturbed(this[kState].body.stream)
     }
   }
 }

package/lib/fetch/headers.js

@@ -11,22 +11,8 @@ const {
   forbiddenResponseHeaderNames
 } = require('./constants')
 
-function binarySearch (arr, val) {
-  let low = 0
-  let high = Math.floor(arr.length / 2)
-
-  while (high > low) {
-    const mid = (high + low) >>> 1
-
-    if (val.localeCompare(arr[mid * 2]) > 0) {
-      low = mid + 1
-    } else {
-      high = mid
-    }
-  }
-
-  return low * 2
-}
+const kHeadersMap = Symbol('headers map')
+const kHeadersSortedMap = Symbol('headers map sorted')
 
 function normalizeAndValidateHeaderName (name) {
   if (name === undefined) {
@@ -91,64 +77,74 @@ function fill (headers, object) {
   }
 }
 
-// TODO: Composition over inheritence? Or helper methods?
-class HeadersList extends Array {
+class HeadersList {
+  constructor (init) {
+    if (init instanceof HeadersList) {
+      this[kHeadersMap] = new Map(init[kHeadersMap])
+      this[kHeadersSortedMap] = init[kHeadersSortedMap]
+    } else {
+      this[kHeadersMap] = new Map(init)
+      this[kHeadersSortedMap] = null
+    }
+  }
+
   append (name, value) {
+    this[kHeadersSortedMap] = null
+
     const normalizedName = normalizeAndValidateHeaderName(name)
     const normalizedValue = normalizeAndValidateHeaderValue(name, value)
 
-    const index = binarySearch(this, normalizedName)
+    const exists = this[kHeadersMap].get(normalizedName)
 
-    if (this[index] === normalizedName) {
-      this[index + 1] += `, ${normalizedValue}`
+    if (exists) {
+      this[kHeadersMap].set(normalizedName, `${exists}, ${normalizedValue}`)
     } else {
-      this.splice(index, 0, normalizedName, normalizedValue)
+      this[kHeadersMap].set(normalizedName, `${normalizedValue}`)
     }
   }
 
-  delete (name) {
+  set (name, value) {
+    this[kHeadersSortedMap] = null
+
     const normalizedName = normalizeAndValidateHeaderName(name)
+    return this[kHeadersMap].set(normalizedName, value)
+  }
 
-    const index = binarySearch(this, normalizedName)
+  delete (name) {
+    this[kHeadersSortedMap] = null
 
-    if (this[index] === normalizedName) {
-      this.splice(index, 2)
-    }
+    const normalizedName = normalizeAndValidateHeaderName(name)
+    return this[kHeadersMap].delete(normalizedName)
   }
 
   get (name) {
     const normalizedName = normalizeAndValidateHeaderName(name)
-
-    const index = binarySearch(this, normalizedName)
-
-    if (this[index] === normalizedName) {
-      return this[index + 1]
-    }
-
-    return null
+    return this[kHeadersMap].get(normalizedName) ?? null
   }
 
   has (name) {
     const normalizedName = normalizeAndValidateHeaderName(name)
+    return this[kHeadersMap].has(normalizedName)
+  }
 
-    const index = binarySearch(this, normalizedName)
+  keys () {
+    return this[kHeadersMap].keys()
+  }
 
-    return this[index] === normalizedName
+  values () {
+    return this[kHeadersMap].values()
   }
 
-  set (name, value) {
-    const normalizedName = normalizeAndValidateHeaderName(name)
-    const normalizedValue = normalizeAndValidateHeaderValue(name, value)
+  entries () {
+    return this[kHeadersMap].entries()
+  }
 
-    const index = binarySearch(this, normalizedName)
-    if (this[index] === normalizedName) {
-      this[index + 1] = normalizedValue
-    } else {
-      this.splice(index, 0, normalizedName, normalizedValue)
-    }
+  [Symbol.iterator] () {
+    return this[kHeadersMap][Symbol.iterator]()
   }
 }
 
+// https://fetch.spec.whatwg.org/#headers-class
 class Headers {
   constructor (...args) {
     if (
@@ -161,7 +157,6 @@ class Headers {
       )
     }
     const init = args.length >= 1 ? args[0] ?? {} : {}
-
     this[kHeadersList] = new HeadersList()
 
     // The new Headers(init) constructor steps are:
@@ -287,20 +282,18 @@ class Headers {
       )
     }
 
-    const normalizedName = normalizeAndValidateHeaderName(String(args[0]))
-
     if (this[kGuard] === 'immutable') {
       throw new TypeError('immutable')
     } else if (
       this[kGuard] === 'request' &&
-      forbiddenHeaderNames.includes(normalizedName)
+      forbiddenHeaderNames.includes(String(args[0]).toLocaleLowerCase())
     ) {
       return
     } else if (this[kGuard] === 'request-no-cors') {
       // TODO
     } else if (
       this[kGuard] === 'response' &&
-      forbiddenResponseHeaderNames.includes(normalizedName)
+      forbiddenResponseHeaderNames.includes(String(args[0]).toLocaleLowerCase())
     ) {
       return
     }
@@ -308,25 +301,41 @@ class Headers {
     return this[kHeadersList].set(String(args[0]), String(args[1]))
   }
 
-  * keys () {
-    const clone = this[kHeadersList].slice()
-    for (let index = 0; index < clone.length; index += 2) {
-      yield clone[index]
+  get [kHeadersSortedMap] () {
+    this[kHeadersList][kHeadersSortedMap] ??= new Map([...this[kHeadersList]].sort((a, b) => a[0] < b[0] ? -1 : 1))
+    return this[kHeadersList][kHeadersSortedMap]
+  }
+
+  keys () {
+    if (!(this instanceof Headers)) {
+      throw new TypeError('Illegal invocation')
     }
+
+    return this[kHeadersSortedMap].keys()
   }
 
-  * values () {
-    const clone = this[kHeadersList].slice()
-    for (let index = 1; index < clone.length; index += 2) {
-      yield clone[index]
+  values () {
+    if (!(this instanceof Headers)) {
+      throw new TypeError('Illegal invocation')
     }
+
+    return this[kHeadersSortedMap].values()
   }
 
-  * entries () {
-    const clone = this[kHeadersList].slice()
-    for (let index = 0; index < clone.length; index += 2) {
-      yield [clone[index], clone[index + 1]]
+  entries () {
+    if (!(this instanceof Headers)) {
+      throw new TypeError('Illegal invocation')
     }
+
+    return this[kHeadersSortedMap].entries()
+  }
+
+  [Symbol.iterator] () {
+    if (!(this instanceof Headers)) {
+      throw new TypeError('Illegal invocation')
+    }
+
+    return this[kHeadersSortedMap]
   }
 
   forEach (...args) {
@@ -346,15 +355,9 @@ class Headers {
     const callback = args[0]
     const thisArg = args[1]
 
-    const clone = this[kHeadersList].slice()
-    for (let index = 0; index < clone.length; index += 2) {
-      callback.call(
-        thisArg,
-        clone[index + 1],
-        clone[index],
-        this
-      )
-    }
+    this[kHeadersSortedMap].forEach((value, index) => {
+      callback.apply(thisArg, [value, index, this])
+    })
   }
 
   [Symbol.for('nodejs.util.inspect.custom')] () {
@@ -384,7 +387,6 @@ module.exports = {
   fill,
   Headers,
   HeadersList,
-  binarySearch,
   normalizeAndValidateHeaderName,
   normalizeAndValidateHeaderValue
 }

package/lib/fetch/index.js

@@ -768,7 +768,7 @@ async function schemeFetch (fetchParams) {
   const {
     protocol: scheme,
     pathname: path
-  } = new URL(requestCurrentURL(request))
+  } = requestCurrentURL(request)
 
   // switch on request’s current URL’s scheme, and run the associated steps:
   switch (scheme) {
@@ -780,7 +780,7 @@ async function schemeFetch (fetchParams) {
         const resp = makeResponse({
           statusText: 'OK',
           headersList: [
-            'content-type', 'text/html;charset=utf-8'
+            ['content-type', 'text/html;charset=utf-8']
           ]
         })
 
@@ -792,7 +792,7 @@ async function schemeFetch (fetchParams) {
       return makeNetworkError('invalid path called')
     }
     case 'blob:': {
-      resolveObjectURL ??= require('buffer').resolveObjectURL
+      resolveObjectURL = resolveObjectURL || require('buffer').resolveObjectURL
 
       // 1. Run these steps, but abort when the ongoing fetch is terminated:
       //    1. Let blob be request’s current URL’s blob URL entry’s object.
@@ -871,7 +871,7 @@ async function schemeFetch (fetchParams) {
       return makeResponse({
         statusText: 'OK',
         headersList: [
-          'content-type', contentType
+          ['content-type', contentType]
         ],
         body: extractBody(dataURLStruct.body)[0]
       })
@@ -1919,8 +1919,10 @@ async function httpNetworkFetch (
         origin: url.origin,
         method: request.method,
         body: fetchParams.controller.dispatcher[kIsMockActive] ? request.body && request.body.source : body,
-        headers: request.headersList,
-        maxRedirections: 0
+        headers: [...request.headersList].flat(),
+        maxRedirections: 0,
+        bodyTimeout: 300_000,
+        headersTimeout: 300_000
       },
       {
         body: null,
@@ -1962,16 +1964,18 @@ async function httpNetworkFetch (
           const decoders = []
 
           // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding
-          for (const coding of codings) {
-            if (/(x-)?gzip/.test(coding)) {
-              decoders.push(zlib.createGunzip())
-            } else if (/(x-)?deflate/.test(coding)) {
-              decoders.push(zlib.createInflate())
-            } else if (coding === 'br') {
-              decoders.push(zlib.createBrotliDecompress())
-            } else {
-              decoders.length = 0
-              break
+          if (request.method !== 'HEAD' && request.method !== 'CONNECT' && !nullBodyStatus.includes(status)) {
+            for (const coding of codings) {
+              if (/(x-)?gzip/.test(coding)) {
+                decoders.push(zlib.createGunzip())
+              } else if (/(x-)?deflate/.test(coding)) {
+                decoders.push(zlib.createInflate())
+              } else if (coding === 'br') {
+                decoders.push(zlib.createBrotliDecompress())
+              } else {
+                decoders.length = 0
+                break
+              }
             }
           }
 
@@ -2029,7 +2033,7 @@ async function httpNetworkFetch (
 
           fetchParams.controller.terminate(error)
 
-          reject(makeNetworkError(error))
+          reject(error)
         }
       }
     ))

package/lib/fetch/request.js

@@ -420,7 +420,15 @@ class Request {
       // 4. If headers is a Headers object, then for each header in its header
       // list, append header’s name/header’s value to this’s headers.
       if (headers instanceof Headers) {
-        this[kState].headersList.push(...headers[kHeadersList])
+        // TODO (fix): Why doesn't this work?
+        // for (const [key, val] of headers[kHeadersList]) {
+        //   this[kHeaders].append(key, val)
+        // }
+
+        this[kState].headersList = new HeadersList([
+          ...this[kState].headersList,
+          ...headers[kHeadersList]
+        ])
       } else {
         // 5. Otherwise, fill this’s headers with headers.
         fillHeaders(this[kState].headersList, headers)
@@ -460,6 +468,7 @@ class Request {
       // this’s headers.
       if (contentType && !this[kHeaders].has('content-type')) {
         this[kHeaders].append('content-type', contentType)
+        this[kState].headersList.append('content-type', contentType)
       }
     }
 
@@ -793,9 +802,8 @@ function makeRequest (init) {
     timingAllowFailed: false,
     ...init,
     headersList: init.headersList
-      ? new HeadersList(...init.headersList)
-      : new HeadersList(),
-    urlList: init.urlList ? [...init.urlList.map((url) => new URL(url))] : []
+      ? new HeadersList(init.headersList)
+      : new HeadersList()
   }
   request.url = request.urlList[0]
   return request

package/lib/fetch/response.js

@@ -81,7 +81,7 @@ class Response {
     const value = parsedURL.toString()
 
     // 7. Append `Location`/value to responseObject’s response’s header list.
-    responseObject[kState].headersList.push('location', value)
+    responseObject[kState].headersList.append('location', value)
 
     // 8. Return responseObject.
     return responseObject
@@ -172,7 +172,7 @@ class Response {
       // not contain `Content-Type`, then append `Content-Type`/Content-Type
       // to this’s response’s header list.
       if (contentType && !this.headers.has('content-type')) {
-        this.headers.set('content-type', contentType)
+        this.headers.append('content-type', contentType)
       }
     }
   }
@@ -350,7 +350,7 @@ function makeResponse (init) {
     statusText: '',
     ...init,
     headersList: init.headersList
-      ? new HeadersList(...init.headersList)
+      ? new HeadersList(init.headersList)
       : new HeadersList(),
     urlList: init.urlList ? [...init.urlList] : []
   }
@@ -393,17 +393,15 @@ function makeFilteredHeadersList (headersList, filter) {
     get (target, prop) {
       // Override methods used by Headers class.
       if (prop === 'get' || prop === 'has') {
-        return (name) => filter(name) ? target[prop](name) : undefined
-      } else if (prop === 'slice') {
-        return (...args) => {
-          assert(args.length === 0)
-          const arr = []
-          for (let index = 0; index < target.length; index += 2) {
-            if (filter(target[index])) {
-              arr.push(target[index], target[index + 1])
+        const defaultReturn = prop === 'has' ? false : null
+        return (name) => filter(name) ? target[prop](name) : defaultReturn
+      } else if (prop === Symbol.iterator) {
+        return function * () {
+          for (const entry of target) {
+            if (filter(entry[0])) {
+              yield entry
             }
           }
-          return arr
         }
       } else {
         return target[prop]
@@ -423,7 +421,10 @@ function filterResponse (response, type) {
 
     return makeFilteredResponse(response, {
       type: 'basic',
-      headersList: makeFilteredHeadersList(response.headersList, (name) => !forbiddenResponseHeaderNames.includes(name))
+      headersList: makeFilteredHeadersList(
+        response.headersList,
+        (name) => !forbiddenResponseHeaderNames.includes(name.toLowerCase())
+      )
     })
   } else if (type === 'cors') {
     // A CORS filtered response is a filtered response whose type is "cors"

package/lib/fetch/util.js

@@ -318,7 +318,42 @@ function sameOrigin (A, B) {
 
 // https://fetch.spec.whatwg.org/#corb-check
 function CORBCheck (request, response) {
-  // TODO
+  // 1. If request’s initiator is "download", then return allowed.
+  if (request.initiator === 'download') {
+    return 'allowed'
+  }
+
+  // 2. If request’s current URL’s scheme is not an HTTP(S) scheme, then return allowed.
+  if (!/^https?$/.test(request.currentURL.scheme)) {
+    return 'allowed'
+  }
+
+  // 3. Let mimeType be the result of extracting a MIME type from response’s header list.
+  const mimeType = response.headersList.get('content-type')
+
+  // 4. If mimeType is failure, then return allowed.
+  if (mimeType === '') {
+    return 'allowed'
+  }
+
+  // 5. If response’s status is 206 and mimeType is a CORB-protected MIME type, then return blocked.
+
+  const isCORBProtectedMIME =
+  (/^text\/html\b/.test(mimeType) ||
+  /^application\/javascript\b/.test(mimeType) ||
+  /^application\/xml\b/.test(mimeType)) && !/^application\/xml\+svg\b/.test(mimeType)
+
+  if (response.status === 206 && isCORBProtectedMIME) {
+    return 'blocked'
+  }
+
+  // 6. If determine nosniff with response’s header list is true and mimeType is a CORB-protected MIME type or its essence is "text/plain", then return blocked.
+  // https://fetch.spec.whatwg.org/#determinenosniff
+  if (response.headersList.get('x-content-type-options') && isCORBProtectedMIME) {
+    return 'blocked'
+  }
+
+  // 7. Return allowed.
   return 'allowed'
 }
 

package/lib/mock/mock-agent.js

@@ -16,8 +16,10 @@ const {
 const MockClient = require('./mock-client')
 const MockPool = require('./mock-pool')
 const { matchValue, buildMockOptions } = require('./mock-utils')
-const { InvalidArgumentError } = require('../core/errors')
+const { InvalidArgumentError, UndiciError } = require('../core/errors')
 const Dispatcher = require('../dispatcher')
+const Pluralizer = require('./pluralizer')
+const PendingInterceptorsFormatter = require('./pending-interceptors-formatter')
 
 class FakeWeakRef {
   constructor (value) {
@@ -134,6 +136,30 @@ class MockAgent extends Dispatcher {
   [kGetNetConnect] () {
     return this[kNetConnect]
   }
+
+  pendingInterceptors () {
+    const mockAgentClients = this[kClients]
+
+    return Array.from(mockAgentClients.entries())
+      .flatMap(([origin, scope]) => scope.deref()[kDispatches].map(dispatch => ({ ...dispatch, origin })))
+      .filter(({ pending }) => pending)
+  }
+
+  assertNoPendingInterceptors ({ pendingInterceptorsFormatter = new PendingInterceptorsFormatter() } = {}) {
+    const pending = this.pendingInterceptors()
+
+    if (pending.length === 0) {
+      return
+    }
+
+    const pluralizer = new Pluralizer('interceptor', 'interceptors').pluralize(pending.length)
+
+    throw new UndiciError(`
+${pluralizer.count} ${pluralizer.noun} ${pluralizer.is} pending:
+
+${pendingInterceptorsFormatter.format(pending)}
+`.trim())
+  }
 }
 
 module.exports = MockAgent

package/lib/mock/mock-interceptor.js

@@ -12,7 +12,7 @@ const {
 const { InvalidArgumentError } = require('../core/errors')
 
 /**
- * Defines the scope API for a interceptor reply
+ * Defines the scope API for an interceptor reply
  */
 class MockScope {
   constructor (mockDispatch) {
@@ -74,6 +74,9 @@ class MockInterceptor {
       const parsedURL = new URL(opts.path, 'data://')
       opts.path = parsedURL.pathname + parsedURL.search
     }
+    if (typeof opts.method === 'string') {
+      opts.method = opts.method.toUpperCase()
+    }
 
     this[kDispatchKey] = buildKey(opts)
     this[kDispatches] = mockDispatches

package/lib/mock/mock-utils.js

@@ -107,9 +107,9 @@ function getMockDispatch (mockDispatches, key) {
 }
 
 function addMockDispatch (mockDispatches, key, data) {
-  const baseData = { times: null, persist: false, consumed: false }
+  const baseData = { timesInvoked: 0, times: 1, persist: false, consumed: false }
   const replyData = typeof data === 'function' ? { callback: data } : { ...data }
-  const newMockDispatch = { ...baseData, ...key, data: { error: null, ...replyData } }
+  const newMockDispatch = { ...baseData, ...key, pending: true, data: { error: null, ...replyData } }
   mockDispatches.push(newMockDispatch)
   return newMockDispatch
 }
@@ -140,6 +140,80 @@ function generateKeyValues (data) {
   return Object.entries(data).reduce((keyValuePairs, [key, value]) => [...keyValuePairs, key, value], [])
 }
 
+/**
+ * @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Status
+ * @param {number} statusCode
+ */
+function getStatusText (statusCode) {
+  switch (statusCode) {
+    case 100: return 'Continue'
+    case 101: return 'Switching Protocols'
+    case 102: return 'Processing'
+    case 103: return 'Early Hints'
+    case 200: return 'OK'
+    case 201: return 'Created'
+    case 202: return 'Accepted'
+    case 203: return 'Non-Authoritative Information'
+    case 204: return 'No Content'
+    case 205: return 'Reset Content'
+    case 206: return 'Partial Content'
+    case 207: return 'Multi-Status'
+    case 208: return 'Already Reported'
+    case 226: return 'IM Used'
+    case 300: return 'Multiple Choice'
+    case 301: return 'Moved Permanently'
+    case 302: return 'Found'
+    case 303: return 'See Other'
+    case 304: return 'Not Modified'
+    case 305: return 'Use Proxy'
+    case 306: return 'unused'
+    case 307: return 'Temporary Redirect'
+    case 308: return 'Permanent Redirect'
+    case 400: return 'Bad Request'
+    case 401: return 'Unauthorized'
+    case 402: return 'Payment Required'
+    case 403: return 'Forbidden'
+    case 404: return 'Not Found'
+    case 405: return 'Method Not Allowed'
+    case 406: return 'Not Acceptable'
+    case 407: return 'Proxy Authentication Required'
+    case 408: return 'Request Timeout'
+    case 409: return 'Conflict'
+    case 410: return 'Gone'
+    case 411: return 'Length Required'
+    case 412: return 'Precondition Failed'
+    case 413: return 'Payload Too Large'
+    case 414: return 'URI Too Large'
+    case 415: return 'Unsupported Media Type'
+    case 416: return 'Range Not Satisfiable'
+    case 417: return 'Expectation Failed'
+    case 418: return 'I\'m a teapot'
+    case 421: return 'Misdirected Request'
+    case 422: return 'Unprocessable Entity'
+    case 423: return 'Locked'
+    case 424: return 'Failed Dependency'
+    case 425: return 'Too Early'
+    case 426: return 'Upgrade Required'
+    case 428: return 'Precondition Required'
+    case 429: return 'Too Many Requests'
+    case 431: return 'Request Header Fields Too Large'
+    case 451: return 'Unavailable For Legal Reasons'
+    case 500: return 'Internal Server Error'
+    case 501: return 'Not Implemented'
+    case 502: return 'Bad Gateway'
+    case 503: return 'Service Unavailable'
+    case 504: return 'Gateway Timeout'
+    case 505: return 'HTTP Version Not Supported'
+    case 506: return 'Variant Also Negotiates'
+    case 507: return 'Insufficient Storage'
+    case 508: return 'Loop Detected'
+    case 510: return 'Not Extended'
+    case 511: return 'Network Authentication Required'
+    default:
+      throw new ReferenceError(`Unknown status code "${statusCode}"!`)
+  }
+}
+
 async function getResponse (body) {
   const buffers = []
   for await (const data of body) {
@@ -156,6 +230,8 @@ function mockDispatch (opts, handler) {
   const key = buildKey(opts)
   const mockDispatch = getMockDispatch(this[kDispatches], key)
 
+  mockDispatch.timesInvoked++
+
   // Here's where we resolve a callback if a callback is present for the dispatch data.
   if (mockDispatch.data.callback) {
     mockDispatch.data = { ...mockDispatch.data, ...mockDispatch.data.callback(opts) }
@@ -163,18 +239,11 @@ function mockDispatch (opts, handler) {
 
   // Parse mockDispatch data
   const { data: { statusCode, data, headers, trailers, error }, delay, persist } = mockDispatch
-  let { times } = mockDispatch
-  if (typeof times === 'number' && times > 0) {
-    times = --mockDispatch.times
-  }
+  const { timesInvoked, times } = mockDispatch
 
-  // If persist is true, skip
-  // Or if times is a number and > 0, skip
-  // Otherwise, mark as consumed
-
-  if (!(persist === true || (typeof times === 'number' && times > 0))) {
-    mockDispatch.consumed = true
-  }
+  // If it's used up and not persistent, mark as consumed
+  mockDispatch.consumed = !persist && timesInvoked >= times
+  mockDispatch.pending = timesInvoked < times
 
   // If specified, trigger dispatch error
   if (error !== null) {
@@ -197,7 +266,7 @@ function mockDispatch (opts, handler) {
     const responseHeaders = generateKeyValues(headers)
     const responseTrailers = generateKeyValues(trailers)
 
-    handler.onHeaders(statusCode, responseHeaders, resume)
+    handler.onHeaders(statusCode, responseHeaders, resume, getStatusText(statusCode))
     handler.onData(Buffer.from(responseData))
     handler.onComplete(responseTrailers)
     deleteMockDispatch(mockDispatches, key)
@@ -264,6 +333,7 @@ module.exports = {
   generateKeyValues,
   matchValue,
   getResponse,
+  getStatusText,
   mockDispatch,
   buildMockDispatch,
   checkNetConnect,

package/lib/mock/pending-interceptors-formatter.js

@@ -0,0 +1,40 @@
+'use strict'
+
+const { Transform } = require('stream')
+const { Console } = require('console')
+
+/**
+ * Gets the output of `console.table(…)` as a string.
+ */
+module.exports = class PendingInterceptorsFormatter {
+  constructor ({ disableColors } = {}) {
+    this.transform = new Transform({
+      transform (chunk, _enc, cb) {
+        cb(null, chunk)
+      }
+    })
+
+    this.logger = new Console({
+      stdout: this.transform,
+      inspectOptions: {
+        colors: !disableColors && !process.env.CI
+      }
+    })
+  }
+
+  format (pendingInterceptors) {
+    const withPrettyHeaders = pendingInterceptors.map(
+      ({ method, path, data: { statusCode }, persist, times, timesInvoked, origin }) => ({
+        Method: method,
+        Origin: origin,
+        Path: path,
+        'Status code': statusCode,
+        Persistent: persist ? '✅' : '❌',
+        Invocations: timesInvoked,
+        Remaining: persist ? Infinity : times - timesInvoked
+      }))
+
+    this.logger.table(withPrettyHeaders)
+    return this.transform.read().toString()
+  }
+}

package/lib/mock/pluralizer.js

@@ -0,0 +1,29 @@
+'use strict'
+
+const singulars = {
+  pronoun: 'it',
+  is: 'is',
+  was: 'was',
+  this: 'this'
+}
+
+const plurals = {
+  pronoun: 'they',
+  is: 'are',
+  was: 'were',
+  this: 'these'
+}
+
+module.exports = class Pluralizer {
+  constructor (singular, plural) {
+    this.singular = singular
+    this.plural = plural
+  }
+
+  pluralize (count) {
+    const one = count === 1
+    const keys = one ? singulars : plurals
+    const noun = one ? this.singular : this.plural
+    return { ...keys, count, noun }
+  }
+}

package/lib/proxy-agent.js

@@ -23,7 +23,7 @@ class ProxyAgent extends DispatcherBase {
         origin: this[kProxy].uri,
         path: opts.origin + opts.path,
         headers: {
-          ...opts.headers,
+          ...buildHeaders(opts.headers),
           host
         }
       },
@@ -55,4 +55,25 @@ function buildProxyOptions (opts) {
   }
 }
 
+/**
+ * @param {string[] | Record<string, string>} headers
+ * @returns {Record<string, string>}
+ */
+function buildHeaders (headers) {
+  // When using undici.fetch, the headers list is stored
+  // as an array.
+  if (Array.isArray(headers)) {
+    /** @type {Record<string, string>} */
+    const headersPair = {}
+
+    for (let i = 0; i < headers.length; i += 2) {
+      headersPair[headers[i]] = headers[i + 1]
+    }
+
+    return headersPair
+  }
+
+  return headers
+}
+
 module.exports = ProxyAgent

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "undici",
-  "version": "5.0.0",
+  "version": "5.1.0",
   "description": "An HTTP/1.1 client, written from scratch for Node.js",
   "homepage": "https://undici.nodejs.org",
   "bugs": {
@@ -40,7 +40,7 @@
     "docs"
   ],
   "scripts": {
-    "build:node": "npx esbuild@0.14.25 index.js --bundle --platform=node --outfile=undici.js",
+    "build:node": "npx esbuild@0.14.38 index-fetch.js --bundle --platform=node --outfile=undici-fetch.js",
     "prebuild:wasm": "docker build -t llhttp_wasm_builder -f build/Dockerfile .",
     "build:wasm": "node build/wasm.js --docker",
     "lint": "standard | snazzy",
@@ -63,22 +63,22 @@
     "fuzz": "jsfuzz test/fuzzing/fuzz.js corpus"
   },
   "devDependencies": {
-    "@sinonjs/fake-timers": "^7.0.5",
-    "@types/node": "^16.9.1",
+    "@sinonjs/fake-timers": "^9.1.2",
+    "@types/node": "^17.0.29",
     "abort-controller": "^3.0.0",
     "busboy": "^0.3.1",
     "chai": "^4.3.4",
     "chai-as-promised": "^7.1.1",
     "chai-iterator": "^3.0.2",
     "chai-string": "^1.5.0",
-    "concurrently": "^6.2.1",
+    "concurrently": "^7.1.0",
     "cronometro": "^0.8.0",
     "delay": "^5.0.0",
     "docsify-cli": "^4.4.3",
     "formdata-node": "^4.3.1",
     "https-pem": "^2.0.0",
     "husky": "^7.0.2",
-    "jest": "^27.2.0",
+    "jest": "^28.0.1",
     "jsfuzz": "^1.0.15",
     "mocha": "^9.1.1",
     "p-timeout": "^3.2.0",
@@ -86,11 +86,11 @@
     "proxy": "^1.0.2",
     "proxyquire": "^2.1.3",
     "semver": "^7.3.5",
-    "sinon": "^11.1.2",
+    "sinon": "^13.0.2",
     "snazzy": "^9.0.0",
-    "standard": "^16.0.3",
-    "tap": "^15.0.9",
-    "tsd": "^0.17.0",
+    "standard": "^17.0.0",
+    "tap": "^16.1.0",
+    "tsd": "^0.20.0",
     "wait-on": "^6.0.0"
   },
   "engines": {

package/types/dispatcher.d.ts

@@ -4,6 +4,7 @@ import { EventEmitter } from 'events'
 import { IncomingHttpHeaders } from 'http'
 import { Blob } from 'buffer'
 import BodyReadable from './readable'
+import { FormData } from './formdata'
 
 type AbortSignal = unknown;
 
@@ -43,7 +44,7 @@ declare namespace Dispatcher {
     path: string;
     method: HttpMethod;
     /** Default: `null` */
-    body?: string | Buffer | Uint8Array | Readable | null;
+    body?: string | Buffer | Uint8Array | Readable | null | FormData;
     /** Default: `null` */
     headers?: IncomingHttpHeaders | string[] | null;
     /** Whether the requests can be safely retried or not. If `false` the request won't be sent until all preceding requests in the pipeline have completed. Default: `true` if `method` is `HEAD` or `GET`. */

package/types/fetch.d.ts

@@ -4,6 +4,7 @@
 
 import { Blob } from 'buffer'
 import { URL, URLSearchParams } from 'url'
+import { ReadableStream } from 'stream/web'
 import { FormData } from './formdata'
 
 export type RequestInfo = string | URL | Request
@@ -13,13 +14,6 @@ export declare function fetch (
   init?: RequestInit
 ): Promise<Response>
 
-declare class ControlledAsyncIterable implements AsyncIterable<Uint8Array> {
-  constructor (input: AsyncIterable<Uint8Array> | Iterable<Uint8Array>)
-  data: AsyncIterable<Uint8Array>
-  disturbed: boolean
-  readonly [Symbol.asyncIterator]: () => AsyncIterator<Uint8Array>
-}
-
 export type BodyInit =
   | ArrayBuffer
   | AsyncIterable<Uint8Array>
@@ -32,7 +26,7 @@ export type BodyInit =
   | string
 
 export interface BodyMixin {
-  readonly body: ControlledAsyncIterable | null
+  readonly body: ReadableStream | null
   readonly bodyUsed: boolean
 
   readonly arrayBuffer: () => Promise<ArrayBuffer>
@@ -139,7 +133,7 @@ export declare class Request implements BodyMixin {
   readonly keepalive: boolean
   readonly signal: AbortSignal
 
-  readonly body: ControlledAsyncIterable | null
+  readonly body: ReadableStream | null
   readonly bodyUsed: boolean
 
   readonly arrayBuffer: () => Promise<ArrayBuffer>
@@ -178,7 +172,7 @@ export declare class Response implements BodyMixin {
   readonly url: string
   readonly redirected: boolean
 
-  readonly body: ControlledAsyncIterable | null
+  readonly body: ReadableStream | null
   readonly bodyUsed: boolean
 
   readonly arrayBuffer: () => Promise<ArrayBuffer>

package/types/mock-agent.d.ts

@@ -1,9 +1,14 @@
 import Agent = require('./agent')
 import Dispatcher = require('./dispatcher')
-import { Interceptable } from './mock-interceptor'
+import { Interceptable, MockInterceptor } from './mock-interceptor'
+import MockDispatch = MockInterceptor.MockDispatch;
 
 export = MockAgent
 
+interface PendingInterceptor extends MockDispatch {
+  origin: string;
+}
+
 /** A mocked Agent class that implements the Agent API. It allows one to intercept HTTP requests made through undici and return mocked responses instead. */
 declare class MockAgent<TMockAgentOptions extends MockAgent.Options = MockAgent.Options> extends Dispatcher {
   constructor(options?: MockAgent.Options)
@@ -26,6 +31,14 @@ declare class MockAgent<TMockAgentOptions extends MockAgent.Options = MockAgent.
   enableNetConnect(host: ((host: string) => boolean)): void;
   /** Causes all requests to throw when requests are not matched in a MockAgent intercept. */
   disableNetConnect(): void;
+  pendingInterceptors(): PendingInterceptor[];
+  assertNoPendingInterceptors(options?: {
+    pendingInterceptorsFormatter?: PendingInterceptorsFormatter;
+  }): void;
+}
+
+interface PendingInterceptorsFormatter {
+  format(pendingInterceptors: readonly PendingInterceptor[]): string;
 }
 
 declare namespace MockAgent {

package/types/mock-interceptor.d.ts

@@ -1,6 +1,6 @@
 import { IncomingHttpHeaders } from 'http'
 import Dispatcher from './dispatcher';
-import { Headers } from './fetch'
+import { BodyInit, Headers } from './fetch'
 
 export {
   Interceptable,
@@ -71,7 +71,7 @@ declare namespace MockInterceptor {
     path: string;
     origin: string;
     method: string;
-    body?: string;
+    body?: BodyInit | Dispatcher.DispatchOptions['body'];
     headers: Headers;
     maxRedirections: number;
   }