undici 5.12.0 → 5.14.0

package/README.md

@@ -178,10 +178,6 @@ Implements [fetch](https://fetch.spec.whatwg.org/#fetch-method).
 
 Only supported on Node 16.8+.
 
-This is [experimental](https://nodejs.org/api/documentation.html#documentation_stability_index) and is not yet fully compliant with the Fetch Standard.
-We plan to ship breaking changes to this feature until it is out of experimental.
-Help us improve the test coverage by following instructions at [nodejs/undici/#951](https://github.com/nodejs/undici/issues/951).
-
 Basic usage example:
 
 ```js
@@ -234,9 +230,15 @@ const data = {
   },
 }
 
-await fetch('https://example.com', { body: data, method: 'POST' })
+await fetch('https://example.com', { body: data, method: 'POST', duplex: 'half' })
 ```
 
+#### `request.duplex`
+
+- half
+
+In this implementation of fetch, `request.duplex` must be set if `request.body` is `ReadableStream` or `Async Iterables`. And fetch requests are currently always be full duplex. More detail refer to [Fetch Standard.](https://fetch.spec.whatwg.org/#dom-requestinit-duplex)
+
 #### `response.body`
 
 Nodejs has two kinds of streams: [web streams](https://nodejs.org/dist/latest-v16.x/docs/api/webstreams.html), which follow the API of the WHATWG web standard found in browsers, and an older Node-specific [streams API](https://nodejs.org/api/stream.html). `response.body` returns a readable web stream. If you would prefer to work with a Node stream you can convert a web stream using `.fromWeb()`.

package/docs/api/Connector.md

@@ -24,8 +24,10 @@ Once you call `buildConnector`, it will return a connector function, which takes
 * **hostname** `string` (required)
 * **host** `string` (optional)
 * **protocol** `string` (required)
-* **port** `number` (required)
+* **port** `string` (required)
 * **servername** `string` (optional)
+* **localAddress** `string | null` (optional) Local address the socket should connect from.
+* **httpSocket** `Socket` (optional) Establish secure connection on a given socket rather than creating a new socket. It can only be sent on TLS update.
 
 ### Basic example
 

package/index.d.ts

@@ -1,19 +1,19 @@
-import Dispatcher = require('./types/dispatcher')
+import Dispatcher from'./types/dispatcher'
 import { setGlobalDispatcher, getGlobalDispatcher } from './types/global-dispatcher'
 import { setGlobalOrigin, getGlobalOrigin } from './types/global-origin'
-import Pool = require('./types/pool')
+import Pool from'./types/pool'
 import { RedirectHandler, DecoratorHandler } from './types/handlers'
 
-import BalancedPool = require('./types/balanced-pool')
-import Client = require('./types/client')
-import buildConnector = require('./types/connector')
-import errors = require('./types/errors')
-import Agent = require('./types/agent')
-import MockClient = require('./types/mock-client')
-import MockPool = require('./types/mock-pool')
-import MockAgent = require('./types/mock-agent')
-import mockErrors = require('./types/mock-errors')
-import ProxyAgent = require('./types/proxy-agent')
+import BalancedPool from './types/balanced-pool'
+import Client from'./types/client'
+import buildConnector from'./types/connector'
+import errors from'./types/errors'
+import Agent from'./types/agent'
+import MockClient from'./types/mock-client'
+import MockPool from'./types/mock-pool'
+import MockAgent from'./types/mock-agent'
+import mockErrors from'./types/mock-errors'
+import ProxyAgent from'./types/proxy-agent'
 import { request, pipeline, stream, connect, upgrade } from './types/api'
 
 export * from './types/fetch'
@@ -27,16 +27,16 @@ export { Dispatcher, BalancedPool, Pool, Client, buildConnector, errors, Agent,
 export default Undici
 
 declare namespace Undici {
-  var Dispatcher: typeof import('./types/dispatcher')
-  var Pool: typeof import('./types/pool');
+  var Dispatcher: typeof import('./types/dispatcher').default
+  var Pool: typeof import('./types/pool').default;
   var RedirectHandler: typeof import ('./types/handlers').RedirectHandler
   var DecoratorHandler: typeof import ('./types/handlers').DecoratorHandler
   var createRedirectInterceptor: typeof import ('./types/interceptors').createRedirectInterceptor
-  var BalancedPool: typeof import('./types/balanced-pool');
-  var Client: typeof import('./types/client');
-  var buildConnector: typeof import('./types/connector');
-  var errors: typeof import('./types/errors');
-  var Agent: typeof import('./types/agent');
+  var BalancedPool: typeof import('./types/balanced-pool').default;
+  var Client: typeof import('./types/client').default;
+  var buildConnector: typeof import('./types/connector').default;
+  var errors: typeof import('./types/errors').default;
+  var Agent: typeof import('./types/agent').default;
   var setGlobalDispatcher: typeof import('./types/global-dispatcher').setGlobalDispatcher;
   var getGlobalDispatcher: typeof import('./types/global-dispatcher').getGlobalDispatcher;
   var request: typeof import('./types/api').request;
@@ -44,9 +44,9 @@ declare namespace Undici {
   var pipeline: typeof import('./types/api').pipeline;
   var connect: typeof import('./types/api').connect;
   var upgrade: typeof import('./types/api').upgrade;
-  var MockClient: typeof import('./types/mock-client');
-  var MockPool: typeof import('./types/mock-pool');
-  var MockAgent: typeof import('./types/mock-agent');
-  var mockErrors: typeof import('./types/mock-errors');
+  var MockClient: typeof import('./types/mock-client').default;
+  var MockPool: typeof import('./types/mock-pool').default;
+  var MockAgent: typeof import('./types/mock-agent').default;
+  var mockErrors: typeof import('./types/mock-errors').default;
   var fetch: typeof import('./types/fetch').fetch;
 }

package/lib/core/connect.js

@@ -4,6 +4,7 @@ const net = require('net')
 const assert = require('assert')
 const util = require('./util')
 const { InvalidArgumentError, ConnectTimeoutError } = require('./errors')
+
 let tls // include tls conditionally since it is not always available
 
 // TODO: session re-use does not wait for the first
@@ -11,15 +12,73 @@ let tls // include tls conditionally since it is not always available
 // resolve the same servername multiple times even when
 // re-use is enabled.
 
+let SessionCache
+if (global.FinalizationRegistry) {
+  SessionCache = class WeakSessionCache {
+    constructor (maxCachedSessions) {
+      this._maxCachedSessions = maxCachedSessions
+      this._sessionCache = new Map()
+      this._sessionRegistry = new global.FinalizationRegistry((key) => {
+        if (this._sessionCache.size < this._maxCachedSessions) {
+          return
+        }
+
+        const ref = this._sessionCache.get(key)
+        if (ref !== undefined && ref.deref() === undefined) {
+          this._sessionCache.delete(key)
+        }
+      })
+    }
+
+    get (sessionKey) {
+      const ref = this._sessionCache.get(sessionKey)
+      return ref ? ref.deref() : null
+    }
+
+    set (sessionKey, session) {
+      if (this._maxCachedSessions === 0) {
+        return
+      }
+
+      this._sessionCache.set(sessionKey, new WeakRef(session))
+      this._sessionRegistry.register(session, sessionKey)
+    }
+  }
+} else {
+  SessionCache = class SimpleSessionCache {
+    constructor (maxCachedSessions) {
+      this._maxCachedSessions = maxCachedSessions
+      this._sessionCache = new Map()
+    }
+
+    get (sessionKey) {
+      return this._sessionCache.get(sessionKey)
+    }
+
+    set (sessionKey, session) {
+      if (this._maxCachedSessions === 0) {
+        return
+      }
+
+      if (this._sessionCache.size >= this._maxCachedSessions) {
+        // remove the oldest session
+        const { value: oldestKey } = this._sessionCache.keys().next()
+        this._sessionCache.delete(oldestKey)
+      }
+
+      this._sessionCache.set(sessionKey, session)
+    }
+  }
+}
+
 function buildConnector ({ maxCachedSessions, socketPath, timeout, ...opts }) {
   if (maxCachedSessions != null && (!Number.isInteger(maxCachedSessions) || maxCachedSessions < 0)) {
     throw new InvalidArgumentError('maxCachedSessions must be a positive integer or zero')
   }
 
   const options = { path: socketPath, ...opts }
-  const sessionCache = new Map()
+  const sessionCache = new SessionCache(maxCachedSessions == null ? 100 : maxCachedSessions)
   timeout = timeout == null ? 10e3 : timeout
-  maxCachedSessions = maxCachedSessions == null ? 100 : maxCachedSessions
 
   return function connect ({ hostname, host, protocol, port, servername, localAddress, httpSocket }, callback) {
     let socket
@@ -47,25 +106,9 @@ function buildConnector ({ maxCachedSessions, socketPath, timeout, ...opts }) {
 
       socket
         .on('session', function (session) {
-          // cache is disabled
-          if (maxCachedSessions === 0) {
-            return
-          }
-
-          if (sessionCache.size >= maxCachedSessions) {
-            // remove the oldest session
-            const { value: oldestKey } = sessionCache.keys().next()
-            sessionCache.delete(oldestKey)
-          }
-
+          // TODO (fix): Can a session become invalid once established? Don't think so?
           sessionCache.set(sessionKey, session)
         })
-        .on('error', function (err) {
-          if (sessionKey && err.code !== 'UND_ERR_INFO') {
-            // TODO (fix): Only delete for session related errors.
-            sessionCache.delete(sessionKey)
-          }
-        })
     } else {
       assert(!httpSocket, 'httpSocket can only be sent on TLS update')
       socket = net.connect({

package/lib/fetch/body.js

@@ -2,22 +2,24 @@
 
 const Busboy = require('busboy')
 const util = require('../core/util')
-const { ReadableStreamFrom, toUSVString, isBlobLike, isReadableStreamLike, readableStreamClose } = require('./util')
+const { ReadableStreamFrom, isBlobLike, isReadableStreamLike, readableStreamClose } = require('./util')
 const { FormData } = require('./formdata')
 const { kState } = require('./symbols')
 const { webidl } = require('./webidl')
 const { DOMException, structuredClone } = require('./constants')
-const { Blob } = require('buffer')
+const { Blob, File: NativeFile } = require('buffer')
 const { kBodyUsed } = require('../core/symbols')
 const assert = require('assert')
 const { isErrored } = require('../core/util')
 const { isUint8Array, isArrayBuffer } = require('util/types')
-const { File } = require('./file')
+const { File: UndiciFile } = require('./file')
 const { StringDecoder } = require('string_decoder')
 const { parseMIMEType, serializeAMimeType } = require('./dataURL')
 
-/** @type {globalThis['ReadableStream']} */
-let ReadableStream
+let ReadableStream = globalThis.ReadableStream
+
+/** @type {globalThis['File']} */
+const File = NativeFile ?? UndiciFile
 
 // https://fetch.spec.whatwg.org/#concept-bodyinit-extract
 function extractBody (object, keepalive = false) {
@@ -66,9 +68,13 @@ function extractBody (object, keepalive = false) {
   let type = null
 
   // 10. Switch on object:
-  if (object == null) {
-    // Note: The IDL processor cannot handle this situation. See
-    // https://crbug.com/335871.
+  if (typeof object === 'string') {
+    // Set source to the UTF-8 encoding of object.
+    // Note: setting source to a Uint8Array here breaks some mocking assumptions.
+    source = object
+
+    // Set type to `text/plain;charset=UTF-8`.
+    type = 'text/plain;charset=UTF-8'
   } else if (object instanceof URLSearchParams) {
     // URLSearchParams
 
@@ -126,7 +132,8 @@ function extractBody (object, keepalive = false) {
 
           yield * value.stream()
 
-          yield enc.encode('\r\n')
+          // '\r\n' encoded
+          yield new Uint8Array([13, 10])
         }
       }
 
@@ -137,7 +144,33 @@ function extractBody (object, keepalive = false) {
     source = object
 
     // Set length to unclear, see html/6424 for improving this.
-    // TODO
+    length = (() => {
+      const prefixLength = prefix.length
+      const boundaryLength = boundary.length
+      let bodyLength = 0
+
+      for (const [name, value] of object) {
+        if (typeof value === 'string') {
+          bodyLength +=
+            prefixLength +
+            Buffer.byteLength(`; name="${escape(normalizeLinefeeds(name))}"\r\n\r\n${normalizeLinefeeds(value)}\r\n`)
+        } else {
+          bodyLength +=
+            prefixLength +
+            Buffer.byteLength(`; name="${escape(normalizeLinefeeds(name))}"` + (value.name ? `; filename="${escape(value.name)}"` : '')) +
+            2 + // \r\n
+            `Content-Type: ${
+              value.type || 'application/octet-stream'
+            }\r\n\r\n`.length
+
+          // value is a Blob or File, and \r\n
+          bodyLength += value.size + 2
+        }
+      }
+
+      bodyLength += boundaryLength + 4 // --boundary--
+      return bodyLength
+    })()
 
     // Set type to `multipart/form-data; boundary=`,
     // followed by the multipart/form-data boundary string generated
@@ -157,6 +190,11 @@ function extractBody (object, keepalive = false) {
     if (object.type) {
       type = object.type
     }
+  } else if (object instanceof Uint8Array) {
+    // byte sequence
+
+    // Set source to object.
+    source = object
   } else if (typeof object[Symbol.asyncIterator] === 'function') {
     // If keepalive is true, then throw a TypeError.
     if (keepalive) {
@@ -172,17 +210,10 @@ function extractBody (object, keepalive = false) {
 
     stream =
       object instanceof ReadableStream ? object : ReadableStreamFrom(object)
-  } else {
-    // TODO: byte sequence?
-    // TODO: scalar value string?
-    // TODO: else?
-    source = toUSVString(object)
-    type = 'text/plain;charset=UTF-8'
   }
 
   // 11. If source is a byte sequence, then set action to a
   // step that returns source and length to source’s length.
-  // TODO: What is a "byte sequence?"
   if (typeof source === 'string' || util.isBuffer(source)) {
     length = Buffer.byteLength(source)
   }
@@ -329,9 +360,7 @@ function bodyMixinMethods (instance) {
     },
 
     async formData () {
-      if (!(this instanceof instance)) {
-        throw new TypeError('Illegal invocation')
-      }
+      webidl.brandCheck(this, instance)
 
       throwIfAborted(this[kState])
 
@@ -347,7 +376,10 @@ function bodyMixinMethods (instance) {
         let busboy
 
         try {
-          busboy = Busboy({ headers })
+          busboy = Busboy({
+            headers,
+            defParamCharset: 'utf8'
+          })
         } catch (err) {
           // Error due to headers:
           throw Object.assign(new TypeError(), { cause: err })
@@ -360,7 +392,7 @@ function bodyMixinMethods (instance) {
           const { filename, encoding, mimeType } = info
           const chunks = []
 
-          if (encoding.toLowerCase() === 'base64') {
+          if (encoding === 'base64' || encoding.toLowerCase() === 'base64') {
             let base64chunk = ''
 
             value.on('data', (chunk) => {
@@ -433,7 +465,7 @@ function bodyMixinMethods (instance) {
         throwIfAborted(this[kState])
 
         // Otherwise, throw a TypeError.
-        webidl.errors.exception({
+        throw webidl.errors.exception({
           header: `${instance.name}.formData`,
           message: 'Could not parse content as FormData.'
         })
@@ -450,11 +482,8 @@ function mixinBody (prototype) {
 
 // https://fetch.spec.whatwg.org/#concept-body-consume-body
 async function specConsumeBody (object, type, instance) {
-  if (!(object instanceof instance)) {
-    throw new TypeError('Illegal invocation')
-  }
+  webidl.brandCheck(object, instance)
 
-  // TODO: why is this needed?
   throwIfAborted(object[kState])
 
   // 1. If object is unusable, then return a promise rejected

package/lib/fetch/constants.js

@@ -8,6 +8,17 @@ const nullBodyStatus = [101, 204, 205, 304]
 
 const redirectStatus = [301, 302, 303, 307, 308]
 
+// https://fetch.spec.whatwg.org/#block-bad-port
+const badPorts = [
+  '1', '7', '9', '11', '13', '15', '17', '19', '20', '21', '22', '23', '25', '37', '42', '43', '53', '69', '77', '79',
+  '87', '95', '101', '102', '103', '104', '109', '110', '111', '113', '115', '117', '119', '123', '135', '137',
+  '139', '143', '161', '179', '389', '427', '465', '512', '513', '514', '515', '526', '530', '531', '532',
+  '540', '548', '554', '556', '563', '587', '601', '636', '989', '990', '993', '995', '1719', '1720', '1723',
+  '2049', '3659', '4045', '5060', '5061', '6000', '6566', '6665', '6666', '6667', '6668', '6669', '6697',
+  '10080'
+]
+
+// https://w3c.github.io/webappsec-referrer-policy/#referrer-policies
 const referrerPolicy = [
   '',
   'no-referrer',
@@ -44,6 +55,11 @@ const requestBodyHeader = [
   'content-type'
 ]
 
+// https://fetch.spec.whatwg.org/#enumdef-requestduplex
+const requestDuplex = [
+  'half'
+]
+
 // http://fetch.spec.whatwg.org/#forbidden-method
 const forbiddenMethods = ['CONNECT', 'TRACE', 'TRACK']
 
@@ -108,5 +124,7 @@ module.exports = {
   redirectStatus,
   corsSafeListedMethods,
   nullBodyStatus,
-  safeMethods
+  safeMethods,
+  badPorts,
+  requestDuplex
 }

package/lib/fetch/dataURL.js

@@ -1,6 +1,7 @@
 const assert = require('assert')
 const { atob } = require('buffer')
-const { isValidHTTPToken } = require('./util')
+const { format } = require('url')
+const { isValidHTTPToken, isomorphicDecode } = require('./util')
 
 const encoder = new TextEncoder()
 
@@ -54,7 +55,6 @@ function dataURLProcessor (dataURL) {
   const encodedBody = input.slice(mimeTypeLength + 1)
 
   // 10. Let body be the percent-decoding of encodedBody.
-  /** @type {Uint8Array|string} */
   let body = stringPercentDecode(encodedBody)
 
   // 11. If mimeType ends with U+003B (;), followed by
@@ -62,7 +62,8 @@ function dataURLProcessor (dataURL) {
   // case-insensitive match for "base64", then:
   if (/;(\u0020){0,}base64$/i.test(mimeType)) {
     // 1. Let stringBody be the isomorphic decode of body.
-    const stringBody = decodeURIComponent(new TextDecoder('utf-8').decode(body))
+    const stringBody = isomorphicDecode(body)
+
     // 2. Set body to the forgiving-base64 decode of
     // stringBody.
     body = forgivingBase64(stringBody)
@@ -111,73 +112,7 @@ function dataURLProcessor (dataURL) {
  * @param {boolean} excludeFragment
  */
 function URLSerializer (url, excludeFragment = false) {
-  // 1. Let output be url’s scheme and U+003A (:) concatenated.
-  let output = url.protocol
-
-  // 2. If url’s host is non-null:
-  if (url.host.length > 0) {
-    // 1. Append "//" to output.
-    output += '//'
-
-    // 2. If url includes credentials, then:
-    if (url.username.length > 0 || url.password.length > 0) {
-      // 1. Append url’s username to output.
-      output += url.username
-
-      // 2. If url’s password is not the empty string, then append U+003A (:),
-      // followed by url’s password, to output.
-      if (url.password.length > 0) {
-        output += ':' + url.password
-      }
-
-      // 3. Append U+0040 (@) to output.
-      output += '@'
-    }
-
-    // 3. Append url’s host, serialized, to output.
-    output += decodeURIComponent(url.hostname)
-
-    // 4. If url’s port is non-null, append U+003A (:) followed by url’s port,
-    // serialized, to output.
-    if (url.port.length > 0) {
-      output += ':' + url.port
-    }
-  }
-
-  // 3. If url’s host is null, url does not have an opaque path,
-  // url’s path’s size is greater than 1, and url’s path[0]
-  // is the empty string, then append U+002F (/) followed by
-  // U+002E (.) to output.
-  // Note: This prevents web+demo:/.//not-a-host/ or web+demo:/path/..//not-a-host/,
-  // when parsed and then serialized, from ending up as web+demo://not-a-host/
-  // (they end up as web+demo:/.//not-a-host/).
-  // Undici implementation note: url's path[0] can never be an
-  // empty string, so we have to slightly alter what the spec says.
-  if (
-    url.host.length === 0 &&
-    url.pathname.length > 1 &&
-    url.href.slice(url.protocol.length + 1)[0] === '.'
-  ) {
-    output += '/.'
-  }
-
-  // 4. Append the result of URL path serializing url to output.
-  output += url.pathname
-
-  // 5. If url’s query is non-null, append U+003F (?),
-  // followed by url’s query, to output.
-  if (url.search.length > 0) {
-    output += url.search
-  }
-
-  // 6. If exclude fragment is false and url’s fragment is non-null,
-  // then append U+0023 (#), followed by url’s fragment, to output.
-  if (excludeFragment === false && url.hash.length > 0) {
-    output += url.hash
-  }
-
-  // 7. Return output.
-  return output
+  return format(url, { fragment: !excludeFragment })
 }
 
 // https://infra.spec.whatwg.org/#collect-a-sequence-of-code-points