undici 5.4.0 → 5.6.0

package/README.md

@@ -185,13 +185,12 @@ Help us improve the test coverage by following instructions at [nodejs/undici/#9
 Basic usage example:
 
 ```js
-    import {fetch} from 'undici';
+import { fetch } from 'undici';
 
-    async function fetchJson() {
-        const res = await fetch('https://example.com')
-        const json = await res.json()
-        console.log(json);
-    }
+
+const res = await fetch('https://example.com')
+const json = await res.json()
+console.log(json);
 ```
 
 You can pass an optional dispatcher to `fetch` as:
@@ -235,9 +234,7 @@ const data = {
   },
 };
 
-(async () => {
-  await fetch("https://example.com", { body: data, method: 'POST' });
-})();
+await fetch("https://example.com", { body: data, method: 'POST' });
 ```
 
 #### `response.body`
@@ -245,14 +242,12 @@ const data = {
 Nodejs has two kinds of streams: [web streams](https://nodejs.org/dist/latest-v16.x/docs/api/webstreams.html), which follow the API of the WHATWG web standard found in browsers, and an older Node-specific [streams API](https://nodejs.org/api/stream.html). `response.body` returns a readable web stream. If you would prefer to work with a Node stream you can convert a web stream using `.fromWeb()`.
 
 ```js
-    import {fetch} from 'undici';
-    import {Readable} from 'node:stream';
+import { fetch } from 'undici';
+import { Readable } from 'node:stream';
 
-    async function fetchStream() {
-        const response = await fetch('https://example.com')
-        const readableWebStream = response.body;
-        const readableNodeStream = Readable.fromWeb(readableWebStream);
-    }
+const response = await fetch('https://example.com')
+const readableWebStream = response.body;
+const readableNodeStream = Readable.fromWeb(readableWebStream);
 ```
 
 #### Specification Compliance

package/docs/api/Dispatcher.md

@@ -461,14 +461,14 @@ Arguments:
 * **options** `RequestOptions`
 * **callback** `(error: Error | null, data: ResponseData) => void` (optional)
 
-Returns: `void | Promise<ResponseData>` - Only returns a `Promise` if no `callback` argument was passed
+Returns: `void | Promise<ResponseData>` - Only returns a `Promise` if no `callback` argument was passed.
 
 #### Parameter: `RequestOptions`
 
 Extends: [`DispatchOptions`](#parameter-dispatchoptions)
 
-* **opaque** `unknown` (optional) - Default: `null` - Used for passing through context to `ResponseData`
-* **signal** `AbortSignal | events.EventEmitter | null` (optional) - Default: `null`
+* **opaque** `unknown` (optional) - Default: `null` - Used for passing through context to `ResponseData`.
+* **signal** `AbortSignal | events.EventEmitter | null` (optional) - Default: `null`.
 * **onInfo** `({statusCode: number, headers: Record<string, string | string[]>}) => void | null` (optional) - Default: `null` - Callback collecting all the info headers (HTTP 100-199) received.
 
 The `RequestOptions.method` property should not be value `'CONNECT'`.
@@ -476,7 +476,7 @@ The `RequestOptions.method` property should not be value `'CONNECT'`.
 #### Parameter: `ResponseData`
 
 * **statusCode** `number`
-* **headers** `http.IncomingHttpHeaders`
+* **headers** `http.IncomingHttpHeaders` - Note that all header keys are lower-cased, e. g. `content-type`.
 * **body** `stream.Readable` which also implements [the body mixin from the Fetch Standard](https://fetch.spec.whatwg.org/#body-mixin).
 * **trailers** `Record<string, string>` - This object starts out
   as empty and will be mutated to contain trailers after `body` has emitted `'end'`.
@@ -497,6 +497,8 @@ The `RequestOptions.method` property should not be value `'CONNECT'`.
 
 - `dump({ limit: Integer })`, dump the response by reading up to `limit` bytes without killing the socket (optional) - Default: 262144.
 
+Note that body will still be a `Readable` even if it is empty, but attempting to deserialize it with `json()` will result in an exception. Recommended way to ensure there is a body to deserialize is to check if status code is not 204, and `content-type` header starts with `application/json`.
+
 #### Example 1 - Basic GET Request
 
 ```js

package/lib/api/api-request.js

@@ -84,7 +84,8 @@ class RequestHandler extends AsyncResource {
     }
 
     const parsedHeaders = util.parseHeaders(rawHeaders)
-    const body = new Readable(resume, abort, parsedHeaders['content-type'])
+    const contentType = parsedHeaders['content-type']
+    const body = new Readable(resume, abort, contentType)
 
     this.callback = null
     this.res = body
@@ -92,8 +93,8 @@ class RequestHandler extends AsyncResource {
 
     if (callback !== null) {
       if (this.throwOnError && statusCode >= 400) {
-        this.runInAsyncScope(callback, null,
-          new ResponseStatusCodeError(`Response status code ${statusCode}${statusMessage ? `: ${statusMessage}` : ''}`, statusCode, headers)
+        this.runInAsyncScope(getResolveErrorBodyCallback, null,
+          { callback, body, contentType, statusCode, statusMessage, headers }
         )
         return
       }
@@ -152,6 +153,33 @@ class RequestHandler extends AsyncResource {
   }
 }
 
+async function getResolveErrorBodyCallback ({ callback, body, contentType, statusCode, statusMessage, headers }) {
+  if (statusCode === 204 || !contentType) {
+    body.dump()
+    process.nextTick(callback, new ResponseStatusCodeError(`Response status code ${statusCode}${statusMessage ? `: ${statusMessage}` : ''}`, statusCode, headers))
+    return
+  }
+
+  try {
+    if (contentType.startsWith('application/json')) {
+      const payload = await body.json()
+      process.nextTick(callback, new ResponseStatusCodeError(`Response status code ${statusCode}${statusMessage ? `: ${statusMessage}` : ''}`, statusCode, headers, payload))
+      return
+    }
+
+    if (contentType.startsWith('text/')) {
+      const payload = await body.text()
+      process.nextTick(callback, new ResponseStatusCodeError(`Response status code ${statusCode}${statusMessage ? `: ${statusMessage}` : ''}`, statusCode, headers, payload))
+      return
+    }
+  } catch (err) {
+    // Process in a fallback if error
+  }
+
+  body.dump()
+  process.nextTick(callback, new ResponseStatusCodeError(`Response status code ${statusCode}${statusMessage ? `: ${statusMessage}` : ''}`, statusCode, headers))
+}
+
 function request (opts, callback) {
   if (callback === undefined) {
     return new Promise((resolve, reject) => {

package/lib/client.js

@@ -720,7 +720,7 @@ class Parser {
       }
     }
 
-    if (request.method === 'CONNECT' && statusCode >= 200 && statusCode < 300) {
+    if (request.method === 'CONNECT') {
       assert(client[kRunning] === 1)
       this.upgrade = true
       return 2
@@ -889,10 +889,8 @@ function onParserTimeout (parser) {
 
   /* istanbul ignore else */
   if (timeoutType === TIMEOUT_HEADERS) {
-    if (!socket[kWriting]) {
-      assert(!parser.paused, 'cannot be paused while waiting for headers')
-      util.destroy(socket, new HeadersTimeoutError())
-    }
+    assert(!parser.paused, 'cannot be paused while waiting for headers')
+    util.destroy(socket, new HeadersTimeoutError())
   } else if (timeoutType === TIMEOUT_BODY) {
     if (!parser.paused) {
       util.destroy(socket, new BodyTimeoutError())

package/lib/core/connect.js

@@ -21,7 +21,7 @@ function buildConnector ({ maxCachedSessions, socketPath, timeout, ...opts }) {
   timeout = timeout == null ? 10e3 : timeout
   maxCachedSessions = maxCachedSessions == null ? 100 : maxCachedSessions
 
-  return function connect ({ hostname, host, protocol, port, servername }, callback) {
+  return function connect ({ hostname, host, protocol, port, servername, httpSocket }, callback) {
     let socket
     if (protocol === 'https:') {
       if (!tls) {
@@ -39,6 +39,7 @@ function buildConnector ({ maxCachedSessions, socketPath, timeout, ...opts }) {
         ...options,
         servername,
         session,
+        socket: httpSocket, // upgrade socket connection
         port: port || 443,
         host: hostname
       })
@@ -65,6 +66,7 @@ function buildConnector ({ maxCachedSessions, socketPath, timeout, ...opts }) {
           }
         })
     } else {
+      assert(!httpSocket, 'httpSocket can only be sent on TLS update')
       socket = net.connect({
         highWaterMark: 64 * 1024, // Same as nodejs fs streams.
         ...options,

package/lib/core/errors.js

@@ -1,13 +1,5 @@
 'use strict'
 
-class AbortError extends Error {
-  constructor () {
-    super('The operation was aborted')
-    this.code = 'ABORT_ERR'
-    this.name = 'AbortError'
-  }
-}
-
 class UndiciError extends Error {
   constructor (message) {
     super(message)
@@ -57,12 +49,13 @@ class BodyTimeoutError extends UndiciError {
 }
 
 class ResponseStatusCodeError extends UndiciError {
-  constructor (message, statusCode, headers) {
+  constructor (message, statusCode, headers, body) {
     super(message)
     Error.captureStackTrace(this, ResponseStatusCodeError)
     this.name = 'ResponseStatusCodeError'
     this.message = message || 'Response Status Code Error'
     this.code = 'UND_ERR_RESPONSE_STATUS_CODE'
+    this.body = body
     this.status = statusCode
     this.statusCode = statusCode
     this.headers = headers
@@ -191,7 +184,6 @@ class HTTPParserError extends Error {
 }
 
 module.exports = {
-  AbortError,
   HTTPParserError,
   UndiciError,
   HeadersTimeoutError,

package/lib/core/request.js

@@ -48,7 +48,11 @@ class Request {
   }, handler) {
     if (typeof path !== 'string') {
       throw new InvalidArgumentError('path must be a string')
-    } else if (path[0] !== '/' && !(path.startsWith('http://') || path.startsWith('https://'))) {
+    } else if (
+      path[0] !== '/' &&
+      !(path.startsWith('http://') || path.startsWith('https://')) &&
+      method !== 'CONNECT'
+    ) {
       throw new InvalidArgumentError('path must be an absolute URL or start with a slash')
     }
 
@@ -80,13 +84,12 @@ class Request {
       this.body = null
     } else if (util.isStream(body)) {
       this.body = body
-    } else if (body instanceof DataView) {
-      // TODO: Why is DataView special?
-      this.body = body.buffer.byteLength ? Buffer.from(body.buffer) : null
-    } else if (body instanceof ArrayBuffer || ArrayBuffer.isView(body)) {
-      this.body = body.byteLength ? Buffer.from(body) : null
     } else if (util.isBuffer(body)) {
       this.body = body.byteLength ? body : null
+    } else if (ArrayBuffer.isView(body)) {
+      this.body = body.buffer.byteLength ? Buffer.from(body.buffer, body.byteOffset, body.byteLength) : null
+    } else if (body instanceof ArrayBuffer) {
+      this.body = body.byteLength ? Buffer.from(body) : null
     } else if (typeof body === 'string') {
       this.body = body.length ? Buffer.from(body) : null
     } else if (util.isFormDataLike(body) || util.isIterable(body) || util.isBlobLike(body)) {

package/lib/fetch/body.js

@@ -4,12 +4,13 @@ const util = require('../core/util')
 const { ReadableStreamFrom, toUSVString, isBlobLike } = require('./util')
 const { FormData } = require('./formdata')
 const { kState } = require('./symbols')
+const { webidl } = require('./webidl')
 const { Blob } = require('buffer')
 const { kBodyUsed } = require('../core/symbols')
 const assert = require('assert')
 const { NotSupportedError } = require('../core/errors')
 const { isErrored } = require('../core/util')
-const { isUint8Array } = require('util/types')
+const { isUint8Array, isArrayBuffer } = require('util/types')
 
 let ReadableStream
 
@@ -61,7 +62,7 @@ function extractBody (object, keepalive = false) {
 
     // Set Content-Type to `application/x-www-form-urlencoded;charset=UTF-8`.
     contentType = 'application/x-www-form-urlencoded;charset=UTF-8'
-  } else if (object instanceof ArrayBuffer || ArrayBuffer.isView(object)) {
+  } else if (isArrayBuffer(object) || ArrayBuffer.isView(object)) {
     // BufferSource
 
     if (object instanceof DataView) {
@@ -262,100 +263,135 @@ function cloneBody (body) {
   }
 }
 
-const methods = {
-  async blob () {
-    const chunks = []
+function bodyMixinMethods (instance) {
+  const methods = {
+    async blob () {
+      if (!(this instanceof instance)) {
+        throw new TypeError('Illegal invocation')
+      }
 
-    if (this[kState].body) {
-      if (isUint8Array(this[kState].body)) {
-        chunks.push(this[kState].body)
-      } else {
-        const stream = this[kState].body.stream
+      const chunks = []
 
-        if (util.isDisturbed(stream)) {
-          throw new TypeError('disturbed')
-        }
+      if (this[kState].body) {
+        if (isUint8Array(this[kState].body)) {
+          chunks.push(this[kState].body)
+        } else {
+          const stream = this[kState].body.stream
 
-        if (stream.locked) {
-          throw new TypeError('locked')
-        }
+          if (util.isDisturbed(stream)) {
+            throw new TypeError('disturbed')
+          }
+
+          if (stream.locked) {
+            throw new TypeError('locked')
+          }
 
-        // Compat.
-        stream[kBodyUsed] = true
+          // Compat.
+          stream[kBodyUsed] = true
 
-        for await (const chunk of stream) {
-          chunks.push(chunk)
+          for await (const chunk of stream) {
+            chunks.push(chunk)
+          }
         }
       }
-    }
 
-    return new Blob(chunks, { type: this.headers.get('Content-Type') || '' })
-  },
+      return new Blob(chunks, { type: this.headers.get('Content-Type') || '' })
+    },
 
-  async arrayBuffer () {
-    const blob = await this.blob()
-    return await blob.arrayBuffer()
-  },
+    async arrayBuffer () {
+      if (!(this instanceof instance)) {
+        throw new TypeError('Illegal invocation')
+      }
 
-  async text () {
-    const blob = await this.blob()
-    return toUSVString(await blob.text())
-  },
+      const blob = await this.blob()
+      return await blob.arrayBuffer()
+    },
 
-  async json () {
-    return JSON.parse(await this.text())
-  },
+    async text () {
+      if (!(this instanceof instance)) {
+        throw new TypeError('Illegal invocation')
+      }
+
+      const blob = await this.blob()
+      return toUSVString(await blob.text())
+    },
 
-  async formData () {
-    const contentType = this.headers.get('Content-Type')
-
-    // If mimeType’s essence is "multipart/form-data", then:
-    if (/multipart\/form-data/.test(contentType)) {
-      throw new NotSupportedError('multipart/form-data not supported')
-    } else if (/application\/x-www-form-urlencoded/.test(contentType)) {
-      // Otherwise, if mimeType’s essence is "application/x-www-form-urlencoded", then:
-
-      // 1. Let entries be the result of parsing bytes.
-      let entries
-      try {
-        entries = new URLSearchParams(await this.text())
-      } catch (err) {
-        // istanbul ignore next: Unclear when new URLSearchParams can fail on a string.
-        // 2. If entries is failure, then throw a TypeError.
-        throw Object.assign(new TypeError(), { cause: err })
+    async json () {
+      if (!(this instanceof instance)) {
+        throw new TypeError('Illegal invocation')
       }
 
-      // 3. Return a new FormData object whose entries are entries.
-      const formData = new FormData()
-      for (const [name, value] of entries) {
-        formData.append(name, value)
+      return JSON.parse(await this.text())
+    },
+
+    async formData () {
+      if (!(this instanceof instance)) {
+        throw new TypeError('Illegal invocation')
+      }
+
+      const contentType = this.headers.get('Content-Type')
+
+      // If mimeType’s essence is "multipart/form-data", then:
+      if (/multipart\/form-data/.test(contentType)) {
+        throw new NotSupportedError('multipart/form-data not supported')
+      } else if (/application\/x-www-form-urlencoded/.test(contentType)) {
+        // Otherwise, if mimeType’s essence is "application/x-www-form-urlencoded", then:
+
+        // 1. Let entries be the result of parsing bytes.
+        let entries
+        try {
+          entries = new URLSearchParams(await this.text())
+        } catch (err) {
+          // istanbul ignore next: Unclear when new URLSearchParams can fail on a string.
+          // 2. If entries is failure, then throw a TypeError.
+          throw Object.assign(new TypeError(), { cause: err })
+        }
+
+        // 3. Return a new FormData object whose entries are entries.
+        const formData = new FormData()
+        for (const [name, value] of entries) {
+          formData.append(name, value)
+        }
+        return formData
+      } else {
+        // Otherwise, throw a TypeError.
+        webidl.errors.exception({
+          header: `${instance.name}.formData`,
+          value: 'Could not parse content as FormData.'
+        })
       }
-      return formData
-    } else {
-      // Otherwise, throw a TypeError.
-      throw new TypeError()
     }
   }
+
+  return methods
 }
 
 const properties = {
   body: {
     enumerable: true,
     get () {
+      if (!this || !this[kState]) {
+        throw new TypeError('Illegal invocation')
+      }
+
       return this[kState].body ? this[kState].body.stream : null
     }
   },
   bodyUsed: {
     enumerable: true,
     get () {
+      if (!this || !this[kState]) {
+        throw new TypeError('Illegal invocation')
+      }
+
       return !!this[kState].body && util.isDisturbed(this[kState].body.stream)
     }
   }
 }
 
 function mixinBody (prototype) {
-  Object.assign(prototype, methods)
-  Object.defineProperties(prototype, properties)
+  Object.assign(prototype.prototype, bodyMixinMethods(prototype))
+  Object.defineProperties(prototype.prototype, properties)
 }
 
 module.exports = {

package/lib/fetch/constants.js

@@ -60,7 +60,19 @@ const subresource = [
   ''
 ]
 
+/** @type {globalThis['DOMException']} */
+const DOMException = globalThis.DOMException ?? (() => {
+  // DOMException was only made a global in Node v17.0.0,
+  // but fetch supports >= v16.5.
+  try {
+    atob('~')
+  } catch (err) {
+    return Object.getPrototypeOf(err).constructor
+  }
+})()
+
 module.exports = {
+  DOMException,
   subresource,
   forbiddenMethods,
   requestBodyHeader,

package/lib/fetch/file.js

@@ -1,19 +1,27 @@
 'use strict'
 
 const { Blob } = require('buffer')
+const { types } = require('util')
 const { kState } = require('./symbols')
+const { isBlobLike } = require('./util')
+const { webidl } = require('./webidl')
 
 class File extends Blob {
   constructor (fileBits, fileName, options = {}) {
-    // TODO: argument idl type check
-
     // The File constructor is invoked with two or three parameters, depending
     // on whether the optional dictionary parameter is used. When the File()
     // constructor is invoked, user agents must run the following steps:
+    if (arguments.length < 2) {
+      throw new TypeError('2 arguments required')
+    }
+
+    fileBits = webidl.converters['sequence<BlobPart>'](fileBits)
+    fileName = webidl.converters.USVString(fileName)
+    options = webidl.converters.FilePropertyBag(options)
 
     // 1. Let bytes be the result of processing blob parts given fileBits and
     // options.
-    // TODO
+    // Note: Blob handles this for us
 
     // 2. Let n be the fileName argument to the constructor.
     const n = fileName
@@ -25,17 +33,14 @@ class File extends Blob {
     //    be set to the type dictionary member. If t contains any characters
     //    outside the range U+0020 to U+007E, then set t to the empty string
     //    and return from these substeps.
-    //    TODO
-    const t = options.type
-
     //    2. Convert every character in t to ASCII lowercase.
-    //    TODO
+    // Note: Blob handles both of these steps for us
 
     //    3. If the lastModified member is provided, let d be set to the
     //    lastModified dictionary member. If it is not provided, set d to the
     //    current date and time represented as the number of milliseconds since
     //    the Unix Epoch (which is the equivalent of Date.now() [ECMA-262]).
-    const d = options.lastModified ?? Date.now()
+    const d = options.lastModified
 
     // 4. Return a new File object F such that:
     // F refers to the bytes byte sequence.
@@ -43,9 +48,8 @@ class File extends Blob {
     // F.name is set to n.
     // F.type is set to t.
     // F.lastModified is set to d.
-    // TODO
 
-    super(fileBits, { type: t })
+    super(processBlobParts(fileBits, options), { type: options.type })
     this[kState] = {
       name: n,
       lastModified: d
@@ -190,4 +194,120 @@ class FileLike {
   }
 }
 
-module.exports = { File: globalThis.File ?? File, FileLike }
+webidl.converters.Blob = webidl.interfaceConverter(Blob)
+
+webidl.converters.BlobPart = function (V, opts) {
+  if (webidl.util.Type(V) === 'Object') {
+    if (isBlobLike(V)) {
+      return webidl.converters.Blob(V, { strict: false })
+    }
+
+    return webidl.converters.BufferSource(V, opts)
+  } else {
+    return webidl.converters.USVString(V, opts)
+  }
+}
+
+webidl.converters['sequence<BlobPart>'] = webidl.sequenceConverter(
+  webidl.converters.BlobPart
+)
+
+// https://www.w3.org/TR/FileAPI/#dfn-FilePropertyBag
+webidl.converters.FilePropertyBag = webidl.dictionaryConverter([
+  {
+    key: 'lastModified',
+    converter: webidl.converters['long long'],
+    get defaultValue () {
+      return Date.now()
+    }
+  },
+  {
+    key: 'type',
+    converter: webidl.converters.DOMString,
+    defaultValue: ''
+  },
+  {
+    key: 'endings',
+    converter: (value) => {
+      value = webidl.converters.DOMString(value)
+      value = value.toLowerCase()
+
+      if (value !== 'native') {
+        value = 'transparent'
+      }
+
+      return value
+    },
+    defaultValue: 'transparent'
+  }
+])
+
+/**
+ * @see https://www.w3.org/TR/FileAPI/#process-blob-parts
+ * @param {(NodeJS.TypedArray|Blob|string)[]} parts
+ * @param {{ type: string, endings: string }} options
+ */
+function processBlobParts (parts, options) {
+  // 1. Let bytes be an empty sequence of bytes.
+  /** @type {NodeJS.TypedArray[]} */
+  const bytes = []
+
+  // 2. For each element in parts:
+  for (const element of parts) {
+    // 1. If element is a USVString, run the following substeps:
+    if (typeof element === 'string') {
+      // 1. Let s be element.
+      let s = element
+
+      // 2. If the endings member of options is "native", set s
+      //    to the result of converting line endings to native
+      //    of element.
+      if (options.endings === 'native') {
+        s = convertLineEndingsNative(s)
+      }
+
+      // 3. Append the result of UTF-8 encoding s to bytes.
+      bytes.push(new TextEncoder().encode(s))
+    } else if (
+      types.isAnyArrayBuffer(element) ||
+      types.isTypedArray(element)
+    ) {
+      // 2. If element is a BufferSource, get a copy of the
+      //    bytes held by the buffer source, and append those
+      //    bytes to bytes.
+      if (!element.buffer) { // ArrayBuffer
+        bytes.push(new Uint8Array(element))
+      } else {
+        bytes.push(element.buffer)
+      }
+    } else if (isBlobLike(element)) {
+      // 3. If element is a Blob, append the bytes it represents
+      //    to bytes.
+      bytes.push(element)
+    }
+  }
+
+  // 3. Return bytes.
+  return bytes
+}
+
+/**
+ * @see https://www.w3.org/TR/FileAPI/#convert-line-endings-to-native
+ * @param {string} s
+ */
+function convertLineEndingsNative (s) {
+  // 1. Let native line ending be be the code point U+000A LF.
+  let nativeLineEnding = '\n'
+
+  // 2. If the underlying platform’s conventions are to
+  //    represent newlines as a carriage return and line feed
+  //    sequence, set native line ending to the code point
+  //    U+000D CR followed by the code point U+000A LF.
+  if (process.platform === 'win32') {
+    nativeLineEnding = '\r\n'
+  }
+
+  return s.replace(/\r?\n/g, nativeLineEnding)
+}
+
+module.exports = { File, FileLike }