undici 4.13.0 → 4.15.0

package/README.md

@@ -164,7 +164,9 @@ Implements [fetch](https://fetch.spec.whatwg.org/#fetch-method).
 
 Only supported on Node 16.5+.
 
-This is [experimental](https://nodejs.org/api/documentation.html#documentation_stability_index) and is not yet fully compliant with the Fetch Standard. We plan to ship breaking changes to this feature until it is out of experimental.
+This is [experimental](https://nodejs.org/api/documentation.html#documentation_stability_index) and is not yet fully compliant with the Fetch Standard.
+We plan to ship breaking changes to this feature until it is out of experimental.
+Help us improve the test coverage by following instructions at [nodejs/undici/#951](https://github.com/nodejs/undici/issues/951).
 
 Basic usage example:
 
@@ -178,6 +180,37 @@ Basic usage example:
     }
 ```
 
+
+#### `request.body`
+
+A body can be of the following types:
+
+- ArrayBuffer
+- ArrayBufferView
+- AsyncIterables
+- Blob
+- Iterables
+- String
+- URLSearchParams
+- FormData
+
+In this implementation of fetch, ```request.body``` now accepts ```Async Iterables```. It is not present in the [Fetch Standard.](https://fetch.spec.whatwg.org)
+
+```js
+import { fetch } from "undici";
+
+const data = {
+  async *[Symbol.asyncIterator]() {
+    yield "hello";
+    yield "world";
+  },
+};
+
+(async () => {
+  await fetch("https://example.com", { body: data, method: 'POST' });
+})();
+```
+
 #### `response.body`
 
 Nodejs has two kinds of streams: [web streams](https://nodejs.org/dist/latest-v16.x/docs/api/webstreams.html) which follow the API of the WHATWG web standard found in browsers, and an older Node-specific [streams API](https://nodejs.org/api/stream.html). `response.body` returns a readable web stream. If you would prefer to work with a Node stream you can convert a web stream using `.fromWeb()`.
@@ -299,6 +332,15 @@ aborted.
 * Refs: https://tools.ietf.org/html/rfc2616#section-8.1.2.2
 * Refs: https://tools.ietf.org/html/rfc7230#section-6.3.2
 
+### Manual Redirect
+
+Since it is not possible to manually follow an HTTP redirect on server-side,
+Undici returns the actual response instead of an `opaqueredirect` filtered one
+when invoked with a `manual` redirect. This aligns `fetch()` with the other
+implementations in Deno and Cloudflare Workers.
+
+Refs: https://fetch.spec.whatwg.org/#atomic-http-redirect-handling
+
 ## Collaborators
 
 * [__Daniele Belardi__](https://github.com/dnlup), <https://www.npmjs.com/~dnlup>

package/docs/api/BalancedPool.md

@@ -11,14 +11,15 @@ Requests are not guaranteed to be dispatched in order of invocation.
 Arguments:
 
 * **upstreams** `URL | string | string[]` - It should only include the **protocol, hostname, and port**.
-* **options** `PoolOptions` (optional)
+* **options** `BalancedPoolOptions` (optional)
 
-### Parameter: `PoolOptions`
+### Parameter: `BalancedPoolOptions`
 
-The `PoolOptions` are passed to each of the `Pool` instances being created.
+Extends: [`PoolOptions`](Pool.md#parameter-pooloptions)
 
-See: [`PoolOptions`](Pool.md#parameter-pooloptions)
+* **factory** `(origin: URL, opts: Object) => Dispatcher` - Default: `(origin, opts) => new Pool(origin, opts)`
 
+The `PoolOptions` are passed to each of the `Pool` instances being created.
 ## Instance Properties
 
 ### `BalancedPool.upstreams`
@@ -33,6 +34,10 @@ Implements [Client.closed](Client.md#clientclosed)
 
 Implements [Client.destroyed](Client.md#clientdestroyed)
 
+### `Pool.stats`
+
+Returns [`PoolStats`](PoolStats.md) instance for this pool.
+
 ## Instance Methods
 
 ### `BalancedPool.addUpstream(upstream)`

package/docs/api/Dispatcher.md

@@ -307,6 +307,59 @@ client.dispatch({
 })
 ```
 
+#### Example 3 - Dispatch POST request
+
+```js
+import { createServer } from 'http'
+import { Client } from 'undici'
+import { once } from 'events'
+
+const server = createServer((request, response) => {
+  request.on('data', (data) => {
+    console.log(`Request Data: ${data.toString('utf8')}`)
+    const body = JSON.parse(data)
+    body.message = 'World'
+    response.end(JSON.stringify(body))
+  })
+}).listen()
+
+await once(server, 'listening')
+
+const client = new Client(`http://localhost:${server.address().port}`)
+
+const data = []
+
+client.dispatch({
+  path: '/',
+  method: 'POST',
+  headers: {
+    'content-type': 'application/json'
+  },
+  body: JSON.stringify({ message: 'Hello' })
+}, {
+  onConnect: () => {
+    console.log('Connected!')
+  },
+  onError: (error) => {
+    console.error(error)
+  },
+  onHeaders: (statusCode, headers) => {
+    console.log(`onHeaders | statusCode: ${statusCode} | headers: ${headers}`)
+  },
+  onData: (chunk) => {
+    console.log('onData: chunk received')
+    data.push(chunk)
+  },
+  onComplete: (trailers) => {
+    console.log(`onComplete | trailers: ${trailers}`)
+    const res = Buffer.concat(data).toString('utf8')
+    console.log(`Response Data: ${res}`)
+    client.close()
+    server.close()
+  }
+})
+```
+
 ### `Dispatcher.pipeline(options, handler)`
 
 For easy use with [stream.pipeline](https://nodejs.org/api/stream.html#stream_stream_pipeline_source_transforms_destination_callback). The `handler` argument should return a `Readable` from which the result will be read. Usually it should just return the `body` argument unless some kind of transformation needs to be performed based on e.g. `headers` or `statusCode`. The `handler` should validate the response and save any required state. If there is an error, it should be thrown. The function returns a `Duplex` which writes to the request and reads from the response.
@@ -436,7 +489,7 @@ The `RequestOptions.method` property should not be value `'CONNECT'`.
 - `body`
 - `bodyUsed`
 
-`body` contains the following additional extentions:
+`body` contains the following additional extensions:
 
 - `dump({ limit: Integer })`, dump the response by reading up to `limit` bytes without killing the socket (optional) - Default: 262144.
 

package/docs/api/MockPool.md

@@ -62,7 +62,7 @@ Returns: `MockInterceptor` corresponding to the input options.
 
 We can define the behaviour of an intercepted request with the following options.
 
-* **reply** `(statusCode: number, replyData: string | Buffer | object, responseOptions?: MockResponseOptions) => MockScope` - define a reply for a matching request. Default for `responseOptions` is `{}`.
+* **reply** `(statusCode: number, replyData: string | Buffer | object | MockInterceptor.MockResponseDataHandler, responseOptions?: MockResponseOptions) => MockScope` - define a reply for a matching request. You can define this as a callback to read incoming request data. Default for `responseOptions` is `{}`.
 * **replyWithError** `(error: Error) => MockScope` - define an error for a matching request to throw.
 * **defaultReplyHeaders** `(headers: Record<string, string>) => MockInterceptor` - define default headers to be included in subsequent replies. These are in addition to headers on a specific reply.
 * **defaultReplyTrailers** `(trailers: Record<string, string>) => MockInterceptor` - define default trailers to be included in subsequent replies. These are in addition to trailers on a specific reply.
@@ -113,6 +113,72 @@ for await (const data of body) {
 }
 ```
 
+#### Example - Mocked request using reply data callbacks
+
+```js
+import { MockAgent, setGlobalDispatcher, request } from 'undici'
+
+const mockAgent = new MockAgent()
+setGlobalDispatcher(mockAgent)
+
+const mockPool = mockAgent.get('http://localhost:3000')
+
+mockPool.intercept({
+  path: '/echo',
+  method: 'GET',
+  headers: {
+    'User-Agent': 'undici',
+    Host: 'example.com'
+  }
+}).reply(200, ({ headers }) => ({ message: headers.get('message') }))
+
+const { statusCode, body, headers } = await request('http://localhost:3000', {
+  headers: {
+    message: 'hello world!'
+  }
+})
+
+console.log('response received', statusCode) // response received 200
+console.log('headers', headers) // { 'content-type': 'application/json' }
+
+for await (const data of body) {
+  console.log('data', data.toString('utf8')) // { "message":"hello world!" }
+}
+```
+
+#### Example - Mocked request using reply options callback
+
+```js
+import { MockAgent, setGlobalDispatcher, request } from 'undici'
+
+const mockAgent = new MockAgent()
+setGlobalDispatcher(mockAgent)
+
+const mockPool = mockAgent.get('http://localhost:3000')
+
+mockPool.intercept({
+  path: '/echo',
+  method: 'GET',
+  headers: {
+    'User-Agent': 'undici',
+    Host: 'example.com'
+  }
+}).reply(({ headers }) => ({ statusCode: 200, data: { message: headers.get('message') }})))
+
+const { statusCode, body, headers } = await request('http://localhost:3000', {
+  headers: {
+    message: 'hello world!'
+  }
+})
+
+console.log('response received', statusCode) // response received 200
+console.log('headers', headers) // { 'content-type': 'application/json' }
+
+for await (const data of body) {
+  console.log('data', data.toString('utf8')) // { "message":"hello world!" }
+}
+```
+
 #### Example - Basic Mocked requests with multiple intercepts
 
 ```js
@@ -130,7 +196,7 @@ mockPool.intercept({
 
 mockPool.intercept({
   path: '/hello',
-  method: 'GET'
+  method: 'GET',
 }).reply(200, 'hello')
 
 const result1 = await request('http://localhost:3000/foo')

package/docs/api/Pool.md

@@ -30,6 +30,10 @@ Implements [Client.closed](Client.md#clientclosed)
 
 Implements [Client.destroyed](Client.md#clientdestroyed)
 
+### `Pool.stats`
+
+Returns [`PoolStats`](PoolStats.md) instance for this pool.
+
 ## Instance Methods
 
 ### `Pool.close([callback])`

package/docs/api/PoolStats.md

@@ -0,0 +1,35 @@
+# Class: PoolStats
+
+Aggregate stats for a [Pool](Pool.md) or [BalancedPool](BalancedPool.md).
+
+## `new PoolStats(pool)`
+
+Arguments:
+
+* **pool** `Pool` - Pool or BalancedPool from which to return stats.
+
+## Instance Properties
+
+### `PoolStats.connected`
+
+Number of open socket connections in this pool.
+
+### `PoolStats.free`
+
+Number of open socket connections in this pool that do not have an active request.
+
+### `PoolStats.pending`
+
+Number of pending requests across all clients in this pool.
+
+### `PoolStats.queued`
+
+Number of queued requests across all clients in this pool.
+
+### `PoolStats.running`
+
+Number of currently active requests across all clients in this pool.
+
+### `PoolStats.size`
+
+Number of active, pending, or queued requests across all clients in this pool.

package/index.d.ts

@@ -10,7 +10,7 @@ import MockClient = require('./types/mock-client')
 import MockPool = require('./types/mock-pool')
 import MockAgent = require('./types/mock-agent')
 import mockErrors = require('./types/mock-errors')
-import ProxyAgent from './types/proxy-agent'
+import ProxyAgent = require('./types/proxy-agent')
 import { request, pipeline, stream, connect, upgrade } from './types/api'
 
 export * from './types/fetch'

package/index.js

@@ -94,12 +94,12 @@ module.exports.getGlobalDispatcher = getGlobalDispatcher
 
 if (nodeMajor > 16 || (nodeMajor === 16 && nodeMinor >= 5)) {
   let fetchImpl = null
-  module.exports.fetch = async function fetch (resource, init) {
+  module.exports.fetch = async function fetch (resource) {
     if (!fetchImpl) {
       fetchImpl = require('./lib/fetch')
     }
     const dispatcher = getGlobalDispatcher()
-    return fetchImpl.call(dispatcher, resource, init)
+    return fetchImpl.apply(dispatcher, arguments)
   }
   module.exports.Headers = require('./lib/fetch/headers').Headers
   module.exports.Response = require('./lib/fetch/response').Response

package/lib/api/api-connect.js

@@ -15,7 +15,7 @@ class ConnectHandler extends AsyncResource {
       throw new InvalidArgumentError('invalid callback')
     }
 
-    const { signal, opaque } = opts
+    const { signal, opaque, responseHeaders } = opts
 
     if (signal && typeof signal.on !== 'function' && typeof signal.addEventListener !== 'function') {
       throw new InvalidArgumentError('signal must be an EventEmitter or EventTarget')
@@ -24,6 +24,7 @@ class ConnectHandler extends AsyncResource {
     super('UNDICI_CONNECT')
 
     this.opaque = opaque || null
+    this.responseHeaders = responseHeaders || null
     this.callback = callback
     this.abort = null
 
@@ -43,15 +44,16 @@ class ConnectHandler extends AsyncResource {
     throw new SocketError('bad connect', null)
   }
 
-  onUpgrade (statusCode, headers, socket) {
+  onUpgrade (statusCode, rawHeaders, socket) {
     const { callback, opaque, context } = this
 
     removeSignal(this)
 
     this.callback = null
+    const headers = this.responseHeaders === 'raw' ? util.parseRawHeaders(rawHeaders) : util.parseHeaders(rawHeaders)
     this.runInAsyncScope(callback, null, null, {
       statusCode,
-      headers: util.parseHeaders(headers),
+      headers,
       socket,
       opaque,
       context

package/lib/api/api-pipeline.js

@@ -69,7 +69,7 @@ class PipelineHandler extends AsyncResource {
       throw new InvalidArgumentError('invalid handler')
     }
 
-    const { signal, method, opaque, onInfo } = opts
+    const { signal, method, opaque, onInfo, responseHeaders } = opts
 
     if (signal && typeof signal.on !== 'function' && typeof signal.addEventListener !== 'function') {
       throw new InvalidArgumentError('signal must be an EventEmitter or EventTarget')
@@ -86,6 +86,7 @@ class PipelineHandler extends AsyncResource {
     super('UNDICI_PIPELINE')
 
     this.opaque = opaque || null
+    this.responseHeaders = responseHeaders || null
     this.handler = handler
     this.abort = null
     this.context = null
@@ -156,12 +157,13 @@ class PipelineHandler extends AsyncResource {
     this.context = context
   }
 
-  onHeaders (statusCode, headers, resume) {
+  onHeaders (statusCode, rawHeaders, resume) {
     const { opaque, handler, context } = this
 
     if (statusCode < 200) {
       if (this.onInfo) {
-        this.onInfo({ statusCode, headers: util.parseHeaders(headers) })
+        const headers = this.responseHeaders === 'raw' ? util.parseRawHeaders(rawHeaders) : util.parseHeaders(rawHeaders)
+        this.onInfo({ statusCode, headers })
       }
       return
     }
@@ -171,9 +173,10 @@ class PipelineHandler extends AsyncResource {
     let body
     try {
       this.handler = null
+      const headers = this.responseHeaders === 'raw' ? util.parseRawHeaders(rawHeaders) : util.parseHeaders(rawHeaders)
       body = this.runInAsyncScope(handler, null, {
         statusCode,
-        headers: util.parseHeaders(headers),
+        headers,
         opaque,
         body: this.res,
         context

package/lib/api/api-request.js

@@ -15,7 +15,7 @@ class RequestHandler extends AsyncResource {
       throw new InvalidArgumentError('invalid opts')
     }
 
-    const { signal, method, opaque, body, onInfo } = opts
+    const { signal, method, opaque, body, onInfo, responseHeaders } = opts
 
     try {
       if (typeof callback !== 'function') {
@@ -42,6 +42,7 @@ class RequestHandler extends AsyncResource {
       throw err
     }
 
+    this.responseHeaders = responseHeaders || null
     this.opaque = opaque || null
     this.callback = callback
     this.res = null
@@ -69,25 +70,27 @@ class RequestHandler extends AsyncResource {
     this.context = context
   }
 
-  onHeaders (statusCode, headers, resume) {
+  onHeaders (statusCode, rawHeaders, resume) {
     const { callback, opaque, abort, context } = this
 
     if (statusCode < 200) {
       if (this.onInfo) {
-        this.onInfo({ statusCode, headers: util.parseHeaders(headers) })
+        const headers = this.responseHeaders === 'raw' ? util.parseRawHeaders(rawHeaders) : util.parseHeaders(rawHeaders)
+        this.onInfo({ statusCode, headers })
       }
       return
     }
 
-    const parsedHeaders = util.parseHeaders(headers)
+    const parsedHeaders = util.parseHeaders(rawHeaders)
     const body = new Readable(resume, abort, parsedHeaders['content-type'])
 
     this.callback = null
     this.res = body
+    const headers = this.responseHeaders === 'raw' ? util.parseRawHeaders(rawHeaders) : util.parseHeaders(rawHeaders)
 
     this.runInAsyncScope(callback, null, null, {
       statusCode,
-      headers: parsedHeaders,
+      headers,
       trailers: this.trailers,
       opaque,
       body,

package/lib/api/api-stream.js

@@ -16,7 +16,7 @@ class StreamHandler extends AsyncResource {
       throw new InvalidArgumentError('invalid opts')
     }
 
-    const { signal, method, opaque, body, onInfo } = opts
+    const { signal, method, opaque, body, onInfo, responseHeaders } = opts
 
     try {
       if (typeof callback !== 'function') {
@@ -47,6 +47,7 @@ class StreamHandler extends AsyncResource {
       throw err
     }
 
+    this.responseHeaders = responseHeaders || null
     this.opaque = opaque || null
     this.factory = factory
     this.callback = callback
@@ -75,20 +76,22 @@ class StreamHandler extends AsyncResource {
     this.context = context
   }
 
-  onHeaders (statusCode, headers, resume) {
+  onHeaders (statusCode, rawHeaders, resume) {
     const { factory, opaque, context } = this
 
     if (statusCode < 200) {
       if (this.onInfo) {
-        this.onInfo({ statusCode, headers: util.parseHeaders(headers) })
+        const headers = this.responseHeaders === 'raw' ? util.parseRawHeaders(rawHeaders) : util.parseHeaders(rawHeaders)
+        this.onInfo({ statusCode, headers })
       }
       return
     }
 
     this.factory = null
+    const headers = this.responseHeaders === 'raw' ? util.parseRawHeaders(rawHeaders) : util.parseHeaders(rawHeaders)
     const res = this.runInAsyncScope(factory, null, {
       statusCode,
-      headers: util.parseHeaders(headers),
+      headers,
       opaque,
       context
     })

package/lib/api/api-upgrade.js

@@ -16,7 +16,7 @@ class UpgradeHandler extends AsyncResource {
       throw new InvalidArgumentError('invalid callback')
     }
 
-    const { signal, opaque } = opts
+    const { signal, opaque, responseHeaders } = opts
 
     if (signal && typeof signal.on !== 'function' && typeof signal.addEventListener !== 'function') {
       throw new InvalidArgumentError('signal must be an EventEmitter or EventTarget')
@@ -24,6 +24,7 @@ class UpgradeHandler extends AsyncResource {
 
     super('UNDICI_UPGRADE')
 
+    this.responseHeaders = responseHeaders || null
     this.opaque = opaque || null
     this.callback = callback
     this.abort = null
@@ -45,7 +46,7 @@ class UpgradeHandler extends AsyncResource {
     throw new SocketError('bad upgrade', null)
   }
 
-  onUpgrade (statusCode, headers, socket) {
+  onUpgrade (statusCode, rawHeaders, socket) {
     const { callback, opaque, context } = this
 
     assert.strictEqual(statusCode, 101)
@@ -53,8 +54,9 @@ class UpgradeHandler extends AsyncResource {
     removeSignal(this)
 
     this.callback = null
+    const headers = this.responseHeaders === 'raw' ? util.parseRawHeaders(rawHeaders) : util.parseHeaders(rawHeaders)
     this.runInAsyncScope(callback, null, null, {
-      headers: util.parseHeaders(headers),
+      headers,
       socket,
       opaque,
       context

package/lib/balanced-pool.js

@@ -1,7 +1,8 @@
 'use strict'
 
 const {
-  BalancedPoolMissingUpstreamError
+  BalancedPoolMissingUpstreamError,
+  InvalidArgumentError
 } = require('./core/errors')
 const {
   PoolBase,
@@ -13,11 +14,17 @@ const {
 } = require('./pool-base')
 const Pool = require('./pool')
 const { kUrl } = require('./core/symbols')
+const { parseOrigin } = require('./core/util')
+const kFactory = Symbol('factory')
 
 const kOptions = Symbol('options')
 
+function defaultFactory (origin, opts) {
+  return new Pool(origin, opts);
+}
+
 class BalancedPool extends PoolBase {
-  constructor (upstreams = [], opts = {}) {
+  constructor (upstreams = [], { factory = defaultFactory, ...opts } = {}) {
     super()
 
     this[kOptions] = opts
@@ -26,28 +33,38 @@ class BalancedPool extends PoolBase {
       upstreams = [upstreams]
     }
 
+    if (typeof factory !== 'function') {
+      throw new InvalidArgumentError('factory must be a function.')
+    }
+
+    this[kFactory] = factory
+
     for (const upstream of upstreams) {
       this.addUpstream(upstream)
     }
   }
 
   addUpstream (upstream) {
+    const upstreamOrigin = parseOrigin(upstream).origin
+
     if (this[kClients].find((pool) => (
-      pool[kUrl].origin === upstream &&
+      pool[kUrl].origin === upstreamOrigin &&
       pool.closed !== true &&
       pool.destroyed !== true
     ))) {
       return this
     }
 
-    this[kAddClient](new Pool(upstream, Object.assign({}, this[kOptions])))
+    this[kAddClient](this[kFactory](upstreamOrigin, Object.assign({}, this[kOptions])))
 
     return this
   }
 
   removeUpstream (upstream) {
+    const upstreamOrigin = parseOrigin(upstream).origin
+
     const pool = this[kClients].find((pool) => (
-      pool[kUrl].origin === upstream &&
+      pool[kUrl].origin === upstreamOrigin &&
       pool.closed !== true &&
       pool.destroyed !== true
     ))

package/lib/client.js

@@ -24,9 +24,6 @@ const {
   HTTPParserError
 } = require('./core/errors')
 const buildConnector = require('./core/connect')
-const llhttpWasmData = require('./llhttp/llhttp.wasm.js')
-const llhttpSimdWasmData = require('./llhttp/llhttp_simd.wasm.js')
-
 const {
   kUrl,
   kReset,
@@ -411,9 +408,11 @@ const constants = require('./llhttp/constants')
 const EMPTY_BUF = Buffer.alloc(0)
 
 async function lazyllhttp () {
+  const llhttpWasmData = process.env.JEST_WORKER_ID ? require('./llhttp/llhttp.wasm.js') : undefined
+
   let mod
   try {
-    mod = await WebAssembly.compile(Buffer.from(llhttpWasmData, 'base64'))
+    mod = await WebAssembly.compile(Buffer.from(require('./llhttp/llhttp_simd.wasm.js'), 'base64'))
   } catch (e) {
     /* istanbul ignore next */
 
@@ -421,7 +420,7 @@ async function lazyllhttp () {
     // being enabled, but the occurring of this other error
     // * https://github.com/emscripten-core/emscripten/issues/11495
     // got me to remove that check to avoid breaking Node 12.
-    mod = await WebAssembly.compile(Buffer.from(llhttpSimdWasmData, 'base64'))
+    mod = await WebAssembly.compile(Buffer.from(llhttpWasmData || require('./llhttp/llhttp.wasm.js'), 'base64'))
   }
 
   return await WebAssembly.instantiate(mod, {
@@ -1484,7 +1483,7 @@ function write (client, request) {
 
   let header = `${method} ${path} HTTP/1.1\r\n`
 
-  if (host) {
+  if (typeof host === 'string') {
     header += `host: ${host}\r\n`
   } else {
     header += client[kHostHeader]

package/lib/core/connect.js

@@ -1,10 +1,10 @@
 'use strict'
 
 const net = require('net')
-const tls = require('tls')
 const assert = require('assert')
 const util = require('./util')
 const { InvalidArgumentError, ConnectTimeoutError } = require('./errors')
+let tls // include tls conditionally since it is not always available
 
 // TODO: session re-use does not wait for the first
 // connection to resolve the session and might therefore
@@ -24,6 +24,9 @@ function buildConnector ({ maxCachedSessions, socketPath, timeout, ...opts }) {
   return function connect ({ hostname, host, protocol, port, servername }, callback) {
     let socket
     if (protocol === 'https:') {
+      if (!tls) {
+        tls = require('tls') 
+      }
       servername = servername || options.servername || util.getServerName(host) || null
 
       const sessionKey = servername || hostname