undici 5.20.0 → 5.21.1

package/README.md

@@ -407,7 +407,7 @@ Refs: https://fetch.spec.whatwg.org/#atomic-http-redirect-handling
 
 ## Workarounds
 
-### Network address family autoselection.
+### Network address family autoselection.
 
 If you experience problem when connecting to a remote server that is resolved by your DNS servers to a IPv6 (AAAA record)
 first, there are chances that your local router or ISP might have problem connecting to IPv6 networks. In that case

package/docs/api/ProxyAgent.md

@@ -19,6 +19,7 @@ Extends: [`AgentOptions`](Agent.md#parameter-agentoptions)
 * **uri** `string` (required) - It can be passed either by a string or a object containing `uri` as string.
 * **token** `string` (optional) - It can be passed by a string of token for authentication.
 * **auth** `string` (**deprecated**) - Use token.
+* **clientFactory** `(origin: URL, opts: Object) => Dispatcher` - Default: `(origin, opts) => new Pool(origin, opts)`
 
 Examples:
 
@@ -83,7 +84,8 @@ import { setGlobalDispatcher, request, ProxyAgent } from 'undici';
 
 const proxyAgent = new ProxyAgent({
   uri: 'my.proxy.server',
-  token: 'Bearer xxxx'
+  // token: 'Bearer xxxx'
+  token: `Basic ${Buffer.from('username:password').toString('base64')}`
 });
 setGlobalDispatcher(proxyAgent);
 

package/lib/api/api-stream.js

@@ -1,10 +1,11 @@
 'use strict'
 
-const { finished } = require('stream')
+const { finished, PassThrough } = require('stream')
 const {
   InvalidArgumentError,
   InvalidReturnValueError,
-  RequestAbortedError
+  RequestAbortedError,
+  ResponseStatusCodeError
 } = require('../core/errors')
 const util = require('../core/util')
 const { AsyncResource } = require('async_hooks')
@@ -16,7 +17,7 @@ class StreamHandler extends AsyncResource {
       throw new InvalidArgumentError('invalid opts')
     }
 
-    const { signal, method, opaque, body, onInfo, responseHeaders } = opts
+    const { signal, method, opaque, body, onInfo, responseHeaders, throwOnError } = opts
 
     try {
       if (typeof callback !== 'function') {
@@ -57,6 +58,7 @@ class StreamHandler extends AsyncResource {
     this.trailers = null
     this.body = body
     this.onInfo = onInfo || null
+    this.throwOnError = throwOnError || false
 
     if (util.isStream(body)) {
       body.on('error', (err) => {
@@ -76,8 +78,8 @@ class StreamHandler extends AsyncResource {
     this.context = context
   }
 
-  onHeaders (statusCode, rawHeaders, resume) {
-    const { factory, opaque, context } = this
+  onHeaders (statusCode, rawHeaders, resume, statusMessage) {
+    const { factory, opaque, context, callback } = this
 
     if (statusCode < 200) {
       if (this.onInfo) {
@@ -96,6 +98,32 @@ class StreamHandler extends AsyncResource {
       context
     })
 
+    if (this.throwOnError && statusCode >= 400) {
+      const headers = this.responseHeaders === 'raw' ? util.parseRawHeaders(rawHeaders) : util.parseHeaders(rawHeaders)
+      const chunks = []
+      const pt = new PassThrough()
+      pt
+        .on('data', (chunk) => chunks.push(chunk))
+        .on('end', () => {
+          const payload = Buffer.concat(chunks).toString('utf8')
+          this.runInAsyncScope(
+            callback,
+            null,
+            new ResponseStatusCodeError(
+              `Response status code ${statusCode}${statusMessage ? `: ${statusMessage}` : ''}`,
+              statusCode,
+              headers,
+              payload
+            )
+          )
+        })
+        .on('error', (err) => {
+          this.onError(err)
+        })
+      this.res = pt
+      return
+    }
+
     if (
       !res ||
       typeof res.write !== 'function' ||

package/lib/api/readable.js

@@ -4,7 +4,7 @@
 
 const assert = require('assert')
 const { Readable } = require('stream')
-const { RequestAbortedError, NotSupportedError } = require('../core/errors')
+const { RequestAbortedError, NotSupportedError, InvalidArgumentError } = require('../core/errors')
 const util = require('../core/util')
 const { ReadableStreamFrom, toUSVString } = require('../core/util')
 
@@ -146,15 +146,31 @@ module.exports = class BodyReadable extends Readable {
 
   async dump (opts) {
     let limit = opts && Number.isFinite(opts.limit) ? opts.limit : 262144
+    const signal = opts && opts.signal
+    const abortFn = () => {
+      this.destroy()
+    }
+    if (signal) {
+      if (typeof signal !== 'object' || !('aborted' in signal)) {
+        throw new InvalidArgumentError('signal must be an AbortSignal')
+      }
+      util.throwIfAborted(signal)
+      signal.addEventListener('abort', abortFn, { once: true })
+    }
     try {
       for await (const chunk of this) {
+        util.throwIfAborted(signal)
         limit -= Buffer.byteLength(chunk)
         if (limit < 0) {
           return
         }
       }
     } catch {
-      // Do nothing...
+      util.throwIfAborted(signal)
+    } finally {
+      if (signal) {
+        signal.removeEventListener('abort', abortFn)
+      }
     }
   }
 }

package/lib/client.js

@@ -1,3 +1,5 @@
+// @ts-check
+
 'use strict'
 
 /* global WebAssembly */
@@ -85,7 +87,15 @@ try {
   channels.connected = { hasSubscribers: false }
 }
 
+/**
+ * @type {import('../types/client').default}
+ */
 class Client extends DispatcherBase {
+  /**
+   *
+   * @param {string|URL} url
+   * @param {import('../types/client').Client.Options} options
+   */
   constructor (url, {
     interceptors,
     maxHeaderSize,
@@ -1658,6 +1668,8 @@ class AsyncWriter {
       process.emitWarning(new RequestContentLengthMismatchError())
     }
 
+    socket.cork()
+
     if (bytesWritten === 0) {
       if (!expectsPayload) {
         socket[kReset] = true
@@ -1678,6 +1690,8 @@ class AsyncWriter {
 
     const ret = socket.write(chunk)
 
+    socket.uncork()
+
     request.onBodySent(chunk)
 
     if (!ret) {

package/lib/core/symbols.js

@@ -41,7 +41,7 @@ module.exports = {
   kClient: Symbol('client'),
   kParser: Symbol('parser'),
   kOnDestroyed: Symbol('destroy callbacks'),
-  kPipelining: Symbol('pipelinig'),
+  kPipelining: Symbol('pipelining'),
   kSocket: Symbol('socket'),
   kHostHeader: Symbol('host header'),
   kConnector: Symbol('connector'),

package/lib/core/util.js

@@ -15,7 +15,7 @@ const [nodeMajor, nodeMinor] = process.versions.node.split('.').map(v => Number(
 function nop () {}
 
 function isStream (obj) {
-  return obj && typeof obj.pipe === 'function'
+  return obj && typeof obj === 'object' && typeof obj.pipe === 'function' && typeof obj.on === 'function'
 }
 
 // based on https://github.com/node-fetch/fetch-blob/blob/8ab587d34080de94140b54f07168451e7d0b655e/index.js#L229-L241 (MIT License)
@@ -46,34 +46,40 @@ function buildURL (url, queryParams) {
 function parseURL (url) {
   if (typeof url === 'string') {
     url = new URL(url)
+
+    if (!/^https?:/.test(url.origin || url.protocol)) {
+      throw new InvalidArgumentError('Invalid URL protocol: the URL must start with `http:` or `https:`.')
+    }
+
+    return url
   }
 
   if (!url || typeof url !== 'object') {
-    throw new InvalidArgumentError('invalid url')
+    throw new InvalidArgumentError('Invalid URL: The URL argument must be a non-null object.')
   }
 
   if (url.port != null && url.port !== '' && !Number.isFinite(parseInt(url.port))) {
-    throw new InvalidArgumentError('invalid port')
+    throw new InvalidArgumentError('Invalid URL: port must be a valid integer or a string representation of an integer.')
   }
 
   if (url.path != null && typeof url.path !== 'string') {
-    throw new InvalidArgumentError('invalid path')
+    throw new InvalidArgumentError('Invalid URL path: the path must be a string or null/undefined.')
   }
 
   if (url.pathname != null && typeof url.pathname !== 'string') {
-    throw new InvalidArgumentError('invalid pathname')
+    throw new InvalidArgumentError('Invalid URL pathname: the pathname must be a string or null/undefined.')
   }
 
   if (url.hostname != null && typeof url.hostname !== 'string') {
-    throw new InvalidArgumentError('invalid hostname')
+    throw new InvalidArgumentError('Invalid URL hostname: the hostname must be a string or null/undefined.')
   }
 
   if (url.origin != null && typeof url.origin !== 'string') {
-    throw new InvalidArgumentError('invalid origin')
+    throw new InvalidArgumentError('Invalid URL origin: the origin must be a string or null/undefined.')
   }
 
   if (!/^https?:/.test(url.origin || url.protocol)) {
-    throw new InvalidArgumentError('invalid protocol')
+    throw new InvalidArgumentError('Invalid URL protocol: the URL must start with `http:` or `https:`.')
   }
 
   if (!(url instanceof URL)) {
@@ -375,23 +381,49 @@ function ReadableStreamFrom (iterable) {
 
 // The chunk should be a FormData instance and contains
 // all the required methods.
-function isFormDataLike (chunk) {
-  return (chunk &&
-    chunk.constructor && chunk.constructor.name === 'FormData' &&
-    typeof chunk === 'object' &&
-      (typeof chunk.append === 'function' &&
-        typeof chunk.delete === 'function' &&
-        typeof chunk.get === 'function' &&
-        typeof chunk.getAll === 'function' &&
-        typeof chunk.has === 'function' &&
-        typeof chunk.set === 'function' &&
-        typeof chunk.entries === 'function' &&
-        typeof chunk.keys === 'function' &&
-        typeof chunk.values === 'function' &&
-        typeof chunk.forEach === 'function')
+function isFormDataLike (object) {
+  return (
+    object &&
+    typeof object === 'object' &&
+    typeof object.append === 'function' &&
+    typeof object.delete === 'function' &&
+    typeof object.get === 'function' &&
+    typeof object.getAll === 'function' &&
+    typeof object.has === 'function' &&
+    typeof object.set === 'function' &&
+    object[Symbol.toStringTag] === 'FormData'
   )
 }
 
+function throwIfAborted (signal) {
+  if (!signal) { return }
+  if (typeof signal.throwIfAborted === 'function') {
+    signal.throwIfAborted()
+  } else {
+    if (signal.aborted) {
+      // DOMException not available < v17.0.0
+      const err = new Error('The operation was aborted')
+      err.name = 'AbortError'
+      throw err
+    }
+  }
+}
+
+const hasToWellFormed = !!String.prototype.toWellFormed
+
+/**
+ * @param {string} val
+ */
+function toUSVString (val) {
+  if (hasToWellFormed) {
+    return `${val}`.toWellFormed()
+  } else if (nodeUtil.toUSVString) {
+    return nodeUtil.toUSVString(val)
+  }
+
+  return `${val}`
+}
+
 const kEnumerableProperty = Object.create(null)
 kEnumerableProperty.enumerable = true
 
@@ -401,7 +433,7 @@ module.exports = {
   isDisturbed,
   isErrored,
   isReadable,
-  toUSVString: nodeUtil.toUSVString || ((val) => `${val}`),
+  toUSVString,
   isReadableAborted,
   isBlobLike,
   parseOrigin,
@@ -423,6 +455,7 @@ module.exports = {
   getSocketInfo,
   isFormDataLike,
   buildURL,
+  throwIfAborted,
   nodeMajor,
   nodeMinor,
   nodeHasAutoSelectFamily: nodeMajor > 18 || (nodeMajor === 18 && nodeMinor >= 13)

package/lib/fetch/constants.js

@@ -48,11 +48,17 @@ const requestCache = [
   'only-if-cached'
 ]
 
+// https://fetch.spec.whatwg.org/#request-body-header-name
 const requestBodyHeader = [
   'content-encoding',
   'content-language',
   'content-location',
-  'content-type'
+  'content-type',
+  // See https://github.com/nodejs/undici/issues/2021
+  // 'Content-Length' is a forbidden header name, which is typically
+  // removed in the Headers implementation. However, undici doesn't
+  // filter out headers, so we add it here.
+  'content-length'
 ]
 
 // https://fetch.spec.whatwg.org/#enumdef-requestduplex

package/lib/fetch/dataURL.js

@@ -1,6 +1,5 @@
 const assert = require('assert')
 const { atob } = require('buffer')
-const { format } = require('url')
 const { isValidHTTPToken, isomorphicDecode } = require('./util')
 
 const encoder = new TextEncoder()
@@ -118,7 +117,17 @@ function dataURLProcessor (dataURL) {
  * @param {boolean} excludeFragment
  */
 function URLSerializer (url, excludeFragment = false) {
-  return format(url, { fragment: !excludeFragment })
+  const href = url.href
+
+  if (!excludeFragment) {
+    return href
+  }
+
+  const hash = href.lastIndexOf('#')
+  if (hash === -1) {
+    return href
+  }
+  return href.slice(0, hash)
 }
 
 // https://infra.spec.whatwg.org/#collect-a-sequence-of-code-points

package/lib/fetch/formdata.js

@@ -61,14 +61,7 @@ class FormData {
 
     // The delete(name) method steps are to remove all entries whose name
     // is name from this’s entry list.
-    const next = []
-    for (const entry of this[kState]) {
-      if (entry.name !== name) {
-        next.push(entry)
-      }
-    }
-
-    this[kState] = next
+    this[kState] = this[kState].filter(entry => entry.name !== name)
   }
 
   get (name) {

package/lib/fetch/index.js

@@ -37,7 +37,10 @@ const {
   isErrorLike,
   fullyReadBody,
   readableStreamClose,
-  isomorphicEncode
+  isomorphicEncode,
+  urlIsLocal,
+  urlIsHttpHttpsScheme,
+  urlHasHttpsScheme
 } = require('./util')
 const { kState, kHeaders, kGuard, kRealm, kHeadersCaseInsensitive } = require('./symbols')
 const assert = require('assert')
@@ -272,7 +275,7 @@ function finalizeAndReportTiming (response, initiatorType = 'other') {
   let cacheState = response.cacheState
 
   // 6. If originalURL’s scheme is not an HTTP(S) scheme, then return.
-  if (!/^https?:/.test(originalURL.protocol)) {
+  if (!urlIsHttpHttpsScheme(originalURL)) {
     return
   }
 
@@ -297,7 +300,7 @@ function finalizeAndReportTiming (response, initiatorType = 'other') {
   // capability.
   // TODO: given global’s relevant settings object’s cross-origin isolated
   // capability?
-  response.timingInfo.endTime = coarsenedSharedCurrentTime()
+  timingInfo.endTime = coarsenedSharedCurrentTime()
 
   // 10. Set response’s timing info to timingInfo.
   response.timingInfo = timingInfo
@@ -530,10 +533,7 @@ async function mainFetch (fetchParams, recursive = false) {
 
   // 3. If request’s local-URLs-only flag is set and request’s current URL is
   // not local, then set response to a network error.
-  if (
-    request.localURLsOnly &&
-    !/^(about|blob|data):/.test(requestCurrentURL(request).protocol)
-  ) {
+  if (request.localURLsOnly && !urlIsLocal(requestCurrentURL(request))) {
     response = makeNetworkError('local URLs only')
   }
 
@@ -623,7 +623,7 @@ async function mainFetch (fetchParams, recursive = false) {
       }
 
       // request’s current URL’s scheme is not an HTTP(S) scheme
-      if (!/^https?:/.test(requestCurrentURL(request).protocol)) {
+      if (!urlIsHttpHttpsScheme(requestCurrentURL(request))) {
         // Return a network error.
         return makeNetworkError('URL scheme must be a HTTP(S) scheme')
       }
@@ -1130,7 +1130,7 @@ async function httpRedirectFetch (fetchParams, response) {
 
   // 6. If locationURL’s scheme is not an HTTP(S) scheme, then return a network
   // error.
-  if (!/^https?:/.test(locationURL.protocol)) {
+  if (!urlIsHttpHttpsScheme(locationURL)) {
     return makeNetworkError('URL scheme must be a HTTP(S) scheme')
   }
 
@@ -1205,7 +1205,7 @@ async function httpRedirectFetch (fetchParams, response) {
   // 14. If request’s body is non-null, then set request’s body to the first return
   // value of safely extracting request’s body’s source.
   if (request.body != null) {
-    assert(request.body.source)
+    assert(request.body.source != null)
     request.body = safelyExtractBody(request.body.source)[0]
   }
 
@@ -1399,7 +1399,7 @@ async function httpNetworkOrCacheFetch (
   //    header if httpRequest’s header list contains that header’s name.
   //    TODO: https://github.com/whatwg/fetch/issues/1285#issuecomment-896560129
   if (!httpRequest.headersList.contains('accept-encoding')) {
-    if (/^https:/.test(requestCurrentURL(httpRequest).protocol)) {
+    if (urlHasHttpsScheme(requestCurrentURL(httpRequest))) {
       httpRequest.headersList.append('accept-encoding', 'br, gzip, deflate')
     } else {
       httpRequest.headersList.append('accept-encoding', 'gzip, deflate')
@@ -1845,6 +1845,7 @@ async function httpNetworkFetch (
       // 4. Set bytes to the result of handling content codings given
       // codings and bytes.
       let bytes
+      let isFailure
       try {
         const { done, value } = await fetchParams.controller.next()
 
@@ -1859,6 +1860,10 @@ async function httpNetworkFetch (
           bytes = undefined
         } else {
           bytes = err
+
+          // err may be propagated from the result of calling readablestream.cancel,
+          // which might not be an error. https://github.com/nodejs/undici/issues/2009
+          isFailure = true
         }
       }
 
@@ -1878,7 +1883,7 @@ async function httpNetworkFetch (
       timingInfo.decodedBodySize += bytes?.byteLength ?? 0
 
       // 6. If bytes is failure, then terminate fetchParams’s controller.
-      if (isErrorLike(bytes)) {
+      if (isFailure) {
         fetchParams.controller.terminate(bytes)
         return
       }
@@ -1979,7 +1984,9 @@ async function httpNetworkFetch (
             const val = headersList[n + 1].toString('latin1')
 
             if (key.toLowerCase() === 'content-encoding') {
-              codings = val.split(',').map((x) => x.trim())
+              // https://www.rfc-editor.org/rfc/rfc7231#section-3.1.2.1
+              // "All content-coding values are case-insensitive..."
+              codings = val.toLowerCase().split(',').map((x) => x.trim())
             } else if (key.toLowerCase() === 'location') {
               location = val
             }
@@ -1998,9 +2005,10 @@ async function httpNetworkFetch (
           // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding
           if (request.method !== 'HEAD' && request.method !== 'CONNECT' && !nullBodyStatus.includes(status) && !willFollow) {
             for (const coding of codings) {
-              if (/(x-)?gzip/.test(coding)) {
+              // https://www.rfc-editor.org/rfc/rfc9112.html#section-7.2
+              if (coding === 'x-gzip' || coding === 'gzip') {
                 decoders.push(zlib.createGunzip())
-              } else if (/(x-)?deflate/.test(coding)) {
+              } else if (coding === 'deflate') {
                 decoders.push(zlib.createInflate())
               } else if (coding === 'br') {
                 decoders.push(zlib.createBrotliDecompress())

package/lib/fetch/request.js

@@ -9,7 +9,8 @@ const util = require('../core/util')
 const {
   isValidHTTPToken,
   sameOrigin,
-  normalizeMethod
+  normalizeMethod,
+  makePolicyContainer
 } = require('./util')
 const {
   forbiddenMethods,
@@ -33,6 +34,7 @@ const { setMaxListeners, getEventListeners, defaultMaxListeners } = require('eve
 let TransformStream = globalThis.TransformStream
 
 const kInit = Symbol('init')
+const kAbortController = Symbol('abortController')
 
 const requestFinalizer = new FinalizationRegistry(({ signal, abort }) => {
   signal.removeEventListener('abort', abort)
@@ -51,10 +53,14 @@ class Request {
     input = webidl.converters.RequestInfo(input)
     init = webidl.converters.RequestInit(init)
 
-    // TODO
+    // https://html.spec.whatwg.org/multipage/webappapis.html#environment-settings-object
     this[kRealm] = {
       settingsObject: {
-        baseUrl: getGlobalOrigin()
+        baseUrl: getGlobalOrigin(),
+        get origin () {
+          return this.baseUrl?.origin
+        },
+        policyContainer: makePolicyContainer()
       }
     }
 
@@ -123,12 +129,12 @@ class Request {
     }
 
     // 10. If init["window"] exists and is non-null, then throw a TypeError.
-    if (init.window !== undefined && init.window != null) {
+    if (init.window != null) {
       throw new TypeError(`'window' option '${window}' must be null`)
     }
 
     // 11. If init["window"] exists, then set window to "no-window".
-    if (init.window !== undefined) {
+    if ('window' in init) {
       window = 'no-window'
     }
 
@@ -349,17 +355,30 @@ class Request {
       if (signal.aborted) {
         ac.abort(signal.reason)
       } else {
+        // Keep a strong ref to ac while request object
+        // is alive. This is needed to prevent AbortController
+        // from being prematurely garbage collected.
+        // See, https://github.com/nodejs/undici/issues/1926.
+        this[kAbortController] = ac
+
         const acRef = new WeakRef(ac)
         const abort = function () {
-          acRef.deref()?.abort(this.reason)
+          const ac = acRef.deref()
+          if (ac !== undefined) {
+            ac.abort(this.reason)
+          }
         }
 
-        if (getEventListeners(signal, 'abort').length >= defaultMaxListeners) {
-          setMaxListeners(100, signal)
-        }
+        // Third-party AbortControllers may not work with these.
+        // See, https://github.com/nodejs/undici/pull/1910#issuecomment-1464495619.
+        try {
+          if (getEventListeners(signal, 'abort').length >= defaultMaxListeners) {
+            setMaxListeners(100, signal)
+          }
+        } catch {}
 
         signal.addEventListener('abort', abort, { once: true })
-        requestFinalizer.register(this, { signal, abort })
+        requestFinalizer.register(ac, { signal, abort })
       }
     }
 
@@ -419,7 +438,7 @@ class Request {
     // non-null, and request’s method is `GET` or `HEAD`, then throw a
     // TypeError.
     if (
-      ((init.body !== undefined && init.body != null) || inputBody != null) &&
+      (init.body != null || inputBody != null) &&
       (request.method === 'GET' || request.method === 'HEAD')
     ) {
       throw new TypeError('Request with GET/HEAD method cannot have body.')
@@ -429,7 +448,7 @@ class Request {
     let initBody = null
 
     // 36. If init["body"] exists and is non-null, then:
-    if (init.body !== undefined && init.body != null) {
+    if (init.body != null) {
       // 1. Let Content-Type be null.
       // 2. Set initBody and Content-Type to the result of extracting
       // init["body"], with keepalive set to request’s keepalive.

package/lib/fetch/response.js

@@ -348,9 +348,7 @@ function makeNetworkError (reason) {
     status: 0,
     error: isError
       ? reason
-      : new Error(reason ? String(reason) : reason, {
-        cause: isError ? reason : undefined
-      }),
+      : new Error(reason ? String(reason) : reason),
     aborted: reason && reason.name === 'AbortError'
   })
 }