undici 5.6.1 → 5.8.1

package/README.md

@@ -176,7 +176,7 @@ Implements [fetch](https://fetch.spec.whatwg.org/#fetch-method).
 * https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch
 * https://fetch.spec.whatwg.org/#fetch-method
 
-Only supported on Node 16.5+.
+Only supported on Node 16.8+.
 
 This is [experimental](https://nodejs.org/api/documentation.html#documentation_stability_index) and is not yet fully compliant with the Fetch Standard.
 We plan to ship breaking changes to this feature until it is out of experimental.

package/docs/api/MockAgent.md

@@ -465,7 +465,7 @@ agent.disableNetConnect()
 agent
   .get('https://example.com')
   .intercept({ method: 'GET', path: '/' })
-  .reply(200, '')
+  .reply(200)
 
 const pendingInterceptors = agent.pendingInterceptors()
 // Returns [
@@ -508,7 +508,7 @@ agent.disableNetConnect()
 agent
   .get('https://example.com')
   .intercept({ method: 'GET', path: '/' })
-  .reply(200, '')
+  .reply(200)
 
 agent.assertNoPendingInterceptors()
 // Throws an UndiciError with the following message:

package/docs/best-practices/mocking-request.md

@@ -1,6 +1,6 @@
 # Mocking Request
 
-Undici have its own mocking [utility](../api/MockAgent.md). It allow us to intercept undici HTTP request and return mocked value instead. It can be useful for testing purposes.
+Undici has its own mocking [utility](../api/MockAgent.md). It allow us to intercept undici HTTP requests and return mocked values instead. It can be useful for testing purposes.
 
 Example:
 
@@ -8,7 +8,7 @@ Example:
 // bank.mjs
 import { request } from 'undici'
 
-export async function bankTransfer(recepient, amount) {
+export async function bankTransfer(recipient, amount) {
   const { body } = await request('http://localhost:3000/bank-transfer',
     {
       method: 'POST',
@@ -16,7 +16,7 @@ export async function bankTransfer(recepient, amount) {
         'X-TOKEN-SECRET': 'SuperSecretToken',
       },
       body: JSON.stringify({
-        recepient,
+        recipient,
         amount
       })
     }
@@ -48,7 +48,7 @@ mockPool.intercept({
     'X-TOKEN-SECRET': 'SuperSecretToken',
   },
   body: JSON.stringify({
-    recepient: '1234567890',
+    recipient: '1234567890',
     amount: '100'
   })
 }).reply(200, {
@@ -77,7 +77,7 @@ Explore other MockAgent functionality [here](../api/MockAgent.md)
 
 ## Debug Mock Value
 
-When the interceptor we wrote are not the same undici will automatically call real HTTP request. To debug our mock value use `mockAgent.disableNetConnect()`
+When the interceptor and the request options are not the same, undici will automatically make a real HTTP request. To prevent real requests from being made, use `mockAgent.disableNetConnect()`:
 
 ```js
 const mockAgent = new MockAgent();
@@ -89,7 +89,7 @@ mockAgent.disableNetConnect()
 const mockPool = mockAgent.get('http://localhost:3000');
 
 mockPool.intercept({
-  path: '/bank-tanfer',
+  path: '/bank-transfer',
   method: 'POST',
 }).reply(200, {
   message: 'transaction processed'
@@ -103,7 +103,7 @@ const badRequest = await bankTransfer('1234567890', '100')
 
 ## Reply with data based on request
 
-If the mocked response needs to be dynamically derived from the request parameters, you can provide a function instead of an object to `reply`
+If the mocked response needs to be dynamically derived from the request parameters, you can provide a function instead of an object to `reply`:
 
 ```js
 mockPool.intercept({
@@ -113,7 +113,7 @@ mockPool.intercept({
     'X-TOKEN-SECRET': 'SuperSecretToken',
   },
   body: JSON.stringify({
-    recepient: '1234567890',
+    recipient: '1234567890',
     amount: '100'
   })
 }).reply(200, (opts) => {
@@ -129,7 +129,7 @@ in this case opts will be
 {
   method: 'POST',
   headers: { 'X-TOKEN-SECRET': 'SuperSecretToken' },
-  body: '{"recepient":"1234567890","amount":"100"}',
+  body: '{"recipient":"1234567890","amount":"100"}',
   origin: 'http://localhost:3000',
   path: '/bank-transfer'
 }

package/docs/best-practices/proxy.md

@@ -20,10 +20,10 @@ import { createServer } from 'http'
 import proxy from 'proxy'
 
 const server = await buildServer()
-const proxy = await buildProxy()
+const proxyServer = await buildProxy()
 
 const serverUrl = `http://localhost:${server.address().port}`
-const proxyUrl = `http://localhost:${proxy.address().port}`
+const proxyUrl = `http://localhost:${proxyServer.address().port}`
 
 server.on('request', (req, res) => {
   console.log(req.url) // '/hello?foo=bar'
@@ -47,7 +47,7 @@ console.log(response.statusCode) // 200
 console.log(JSON.parse(data)) // { hello: 'world' }
 
 server.close()
-proxy.close()
+proxyServer.close()
 client.close()
 
 function buildServer () {
@@ -73,12 +73,12 @@ import { createServer } from 'http'
 import proxy from 'proxy'
 
 const server = await buildServer()
-const proxy = await buildProxy()
+const proxyServer = await buildProxy()
 
 const serverUrl = `http://localhost:${server.address().port}`
-const proxyUrl = `http://localhost:${proxy.address().port}`
+const proxyUrl = `http://localhost:${proxyServer.address().port}`
 
-proxy.authenticate = function (req, fn) {
+proxyServer.authenticate = function (req, fn) {
   fn(null, req.headers['proxy-authorization'] === `Basic ${Buffer.from('user:pass').toString('base64')}`)
 }
 
@@ -107,7 +107,7 @@ console.log(response.statusCode) // 200
 console.log(JSON.parse(data)) // { hello: 'world' }
 
 server.close()
-proxy.close()
+proxyServer.close()
 client.close()
 
 function buildServer () {
@@ -124,3 +124,4 @@ function buildProxy () {
   })
 }
 ```
+

package/index.js

@@ -80,7 +80,7 @@ function makeDispatcher (fn) {
 module.exports.setGlobalDispatcher = setGlobalDispatcher
 module.exports.getGlobalDispatcher = getGlobalDispatcher
 
-if (nodeMajor > 16 || (nodeMajor === 16 && nodeMinor >= 5)) {
+if (nodeMajor > 16 || (nodeMajor === 16 && nodeMinor >= 8)) {
   let fetchImpl = null
   module.exports.fetch = async function fetch (resource) {
     if (!fetchImpl) {

package/lib/balanced-pool.js

@@ -18,6 +18,17 @@ const { parseOrigin } = require('./core/util')
 const kFactory = Symbol('factory')
 
 const kOptions = Symbol('options')
+const kGreatestCommonDivisor = Symbol('kGreatestCommonDivisor')
+const kCurrentWeight = Symbol('kCurrentWeight')
+const kIndex = Symbol('kIndex')
+const kWeight = Symbol('kWeight')
+const kMaxWeightPerServer = Symbol('kMaxWeightPerServer')
+const kErrorPenalty = Symbol('kErrorPenalty')
+
+function getGreatestCommonDivisor (a, b) {
+  if (b === 0) return a
+  return getGreatestCommonDivisor(b, a % b)
+}
 
 function defaultFactory (origin, opts) {
   return new Pool(origin, opts)
@@ -28,6 +39,11 @@ class BalancedPool extends PoolBase {
     super()
 
     this[kOptions] = opts
+    this[kIndex] = -1
+    this[kCurrentWeight] = 0
+
+    this[kMaxWeightPerServer] = this[kOptions].maxWeightPerServer || 100
+    this[kErrorPenalty] = this[kOptions].errorPenalty || 15
 
     if (!Array.isArray(upstreams)) {
       upstreams = [upstreams]
@@ -42,6 +58,7 @@ class BalancedPool extends PoolBase {
     for (const upstream of upstreams) {
       this.addUpstream(upstream)
     }
+    this._updateBalancedPoolStats()
   }
 
   addUpstream (upstream) {
@@ -54,12 +71,40 @@ class BalancedPool extends PoolBase {
     ))) {
       return this
     }
+    const pool = this[kFactory](upstreamOrigin, Object.assign({}, this[kOptions]))
+
+    this[kAddClient](pool)
+    pool.on('connect', () => {
+      pool[kWeight] = Math.min(this[kMaxWeightPerServer], pool[kWeight] + this[kErrorPenalty])
+    })
+
+    pool.on('connectionError', () => {
+      pool[kWeight] = Math.max(1, pool[kWeight] - this[kErrorPenalty])
+      this._updateBalancedPoolStats()
+    })
+
+    pool.on('disconnect', (...args) => {
+      const err = args[2]
+      if (err && err.code === 'UND_ERR_SOCKET') {
+        // decrease the weight of the pool.
+        pool[kWeight] = Math.max(1, pool[kWeight] - this[kErrorPenalty])
+        this._updateBalancedPoolStats()
+      }
+    })
+
+    for (const client of this[kClients]) {
+      client[kWeight] = this[kMaxWeightPerServer]
+    }
 
-    this[kAddClient](this[kFactory](upstreamOrigin, Object.assign({}, this[kOptions])))
+    this._updateBalancedPoolStats()
 
     return this
   }
 
+  _updateBalancedPoolStats () {
+    this[kGreatestCommonDivisor] = this[kClients].map(p => p[kWeight]).reduce(getGreatestCommonDivisor, 0)
+  }
+
   removeUpstream (upstream) {
     const upstreamOrigin = parseOrigin(upstream).origin
 
@@ -100,10 +145,42 @@ class BalancedPool extends PoolBase {
       return
     }
 
-    this[kClients].splice(this[kClients].indexOf(dispatcher), 1)
-    this[kClients].push(dispatcher)
+    const allClientsBusy = this[kClients].map(pool => pool[kNeedDrain]).reduce((a, b) => a && b, true)
+
+    if (allClientsBusy) {
+      return
+    }
+
+    let counter = 0
+
+    let maxWeightIndex = this[kClients].findIndex(pool => !pool[kNeedDrain])
+
+    while (counter++ < this[kClients].length) {
+      this[kIndex] = (this[kIndex] + 1) % this[kClients].length
+      const pool = this[kClients][this[kIndex]]
+
+      // find pool index with the largest weight
+      if (pool[kWeight] > this[kClients][maxWeightIndex][kWeight] && !pool[kNeedDrain]) {
+        maxWeightIndex = this[kIndex]
+      }
+
+      // decrease the current weight every `this[kClients].length`.
+      if (this[kIndex] === 0) {
+        // Set the current weight to the next lower weight.
+        this[kCurrentWeight] = this[kCurrentWeight] - this[kGreatestCommonDivisor]
+
+        if (this[kCurrentWeight] <= 0) {
+          this[kCurrentWeight] = this[kMaxWeightPerServer]
+        }
+      }
+      if (pool[kWeight] >= this[kCurrentWeight] && (!pool[kNeedDrain])) {
+        return pool
+      }
+    }
 
-    return dispatcher
+    this[kCurrentWeight] = this[kClients][maxWeightIndex][kWeight]
+    this[kIndex] = maxWeightIndex
+    return this[kClients][maxWeightIndex]
   }
 }
 

package/lib/core/connect.js

@@ -75,14 +75,12 @@ function buildConnector ({ maxCachedSessions, socketPath, timeout, ...opts }) {
       })
     }
 
-    const timeoutId = timeout
-      ? setTimeout(onConnectTimeout, timeout, socket)
-      : null
+    const cancelTimeout = setupTimeout(() => onConnectTimeout(socket), timeout)
 
     socket
       .setNoDelay(true)
       .once(protocol === 'https:' ? 'secureConnect' : 'connect', function () {
-        clearTimeout(timeoutId)
+        cancelTimeout()
 
         if (callback) {
           const cb = callback
@@ -91,7 +89,7 @@ function buildConnector ({ maxCachedSessions, socketPath, timeout, ...opts }) {
         }
       })
       .on('error', function (err) {
-        clearTimeout(timeoutId)
+        cancelTimeout()
 
         if (callback) {
           const cb = callback
@@ -104,6 +102,31 @@ function buildConnector ({ maxCachedSessions, socketPath, timeout, ...opts }) {
   }
 }
 
+function setupTimeout (onConnectTimeout, timeout) {
+  if (!timeout) {
+    return () => {}
+  }
+
+  let s1 = null
+  let s2 = null
+  const timeoutId = setTimeout(() => {
+    // setImmediate is added to make sure that we priotorise socket error events over timeouts
+    s1 = setImmediate(() => {
+      if (process.platform === 'win32') {
+        // Windows needs an extra setImmediate probably due to implementation differences in the socket logic
+        s2 = setImmediate(() => onConnectTimeout())
+      } else {
+        onConnectTimeout()
+      }
+    })
+  }, timeout)
+  return () => {
+    clearTimeout(timeoutId)
+    clearImmediate(s1)
+    clearImmediate(s2)
+  }
+}
+
 function onConnectTimeout (socket) {
   util.destroy(socket, new ConnectTimeoutError())
 }

package/lib/core/request.js

@@ -7,6 +7,27 @@ const {
 const assert = require('assert')
 const util = require('./util')
 
+// tokenRegExp and headerCharRegex have been lifted from
+// https://github.com/nodejs/node/blob/main/lib/_http_common.js
+
+/**
+ * Verifies that the given val is a valid HTTP token
+ * per the rules defined in RFC 7230
+ * See https://tools.ietf.org/html/rfc7230#section-3.2.6
+ */
+const tokenRegExp = /^[\^_`a-zA-Z\-0-9!#$%&'*+.|~]+$/
+
+/**
+ * Matches if val contains an invalid field-vchar
+ *  field-value    = *( field-content / obs-fold )
+ *  field-content  = field-vchar [ 1*( SP / HTAB ) field-vchar ]
+ *  field-vchar    = VCHAR / obs-text
+ */
+const headerCharRegex = /[^\t\x20-\x7e\x80-\xff]/
+
+// Verifies that a given path is valid does not contain control chars \x00 to \x20
+const invalidPathRegex = /[^\u0021-\u00ff]/
+
 const kHandler = Symbol('handler')
 
 const channels = {}
@@ -54,10 +75,14 @@ class Request {
       method !== 'CONNECT'
     ) {
       throw new InvalidArgumentError('path must be an absolute URL or start with a slash')
+    } else if (invalidPathRegex.exec(path) !== null) {
+      throw new InvalidArgumentError('invalid request path')
     }
 
     if (typeof method !== 'string') {
       throw new InvalidArgumentError('method must be a string')
+    } else if (tokenRegExp.exec(method) === null) {
+      throw new InvalidArgumentError('invalid request method')
     }
 
     if (upgrade && typeof upgrade !== 'string') {
@@ -140,8 +165,8 @@ class Request {
     }
 
     if (util.isFormDataLike(this.body)) {
-      if (nodeMajor < 16 || (nodeMajor === 16 && nodeMinor < 5)) {
-        throw new InvalidArgumentError('Form-Data bodies are only supported in node v16.5 and newer.')
+      if (nodeMajor < 16 || (nodeMajor === 16 && nodeMinor < 8)) {
+        throw new InvalidArgumentError('Form-Data bodies are only supported in node v16.8 and newer.')
       }
 
       if (!extractBody) {
@@ -301,6 +326,10 @@ function processHeader (request, key, val) {
     key.toLowerCase() === 'expect'
   ) {
     throw new NotSupportedError('expect header not supported')
+  } else if (tokenRegExp.exec(key) === null) {
+    throw new InvalidArgumentError('invalid header key')
+  } else if (headerCharRegex.exec(val) !== null) {
+    throw new InvalidArgumentError(`invalid ${key} header`)
   } else {
     request.headers += `${key}: ${val}\r\n`
   }

package/lib/fetch/body.js

@@ -15,12 +15,7 @@ const { isUint8Array, isArrayBuffer } = require('util/types')
 let ReadableStream
 
 async function * blobGen (blob) {
-  if (blob.stream) {
-    yield * blob.stream()
-  } else {
-    // istanbul ignore next: node < 16.7
-    yield await blob.arrayBuffer()
-  }
+  yield * blob.stream()
 }
 
 // https://fetch.spec.whatwg.org/#concept-bodyinit-extract
@@ -263,6 +258,29 @@ function cloneBody (body) {
   }
 }
 
+async function * consumeBody (body) {
+  if (body) {
+    if (isUint8Array(body)) {
+      yield body
+    } else {
+      const stream = body.stream
+
+      if (util.isDisturbed(stream)) {
+        throw new TypeError('disturbed')
+      }
+
+      if (stream.locked) {
+        throw new TypeError('locked')
+      }
+
+      // Compat.
+      stream[kBodyUsed] = true
+
+      yield * stream
+    }
+  }
+}
+
 function bodyMixinMethods (instance) {
   const methods = {
     async blob () {
@@ -272,27 +290,14 @@ function bodyMixinMethods (instance) {
 
       const chunks = []
 
-      if (this[kState].body) {
-        if (isUint8Array(this[kState].body)) {
-          chunks.push(this[kState].body)
-        } else {
-          const stream = this[kState].body.stream
-
-          if (util.isDisturbed(stream)) {
-            throw new TypeError('disturbed')
-          }
-
-          if (stream.locked) {
-            throw new TypeError('locked')
-          }
-
-          // Compat.
-          stream[kBodyUsed] = true
-
-          for await (const chunk of stream) {
-            chunks.push(chunk)
-          }
+      for await (const chunk of consumeBody(this[kState].body)) {
+        if (!isUint8Array(chunk)) {
+          throw new TypeError('Expected Uint8Array chunk')
         }
+
+        // Assemble one final large blob with Uint8Array's can exhaust memory.
+        // That's why we create create multiple blob's and using references
+        chunks.push(new Blob([chunk]))
       }
 
       return new Blob(chunks, { type: this.headers.get('Content-Type') || '' })
@@ -303,8 +308,54 @@ function bodyMixinMethods (instance) {
         throw new TypeError('Illegal invocation')
       }
 
-      const blob = await this.blob()
-      return await blob.arrayBuffer()
+      const contentLength = this.headers.get('content-length')
+      const encoded = this.headers.has('content-encoding')
+
+      // if we have content length and no encoding, then we can
+      // pre allocate the buffer and just read the data into it
+      if (!encoded && contentLength) {
+        const buffer = new Uint8Array(contentLength)
+        let offset = 0
+
+        for await (const chunk of consumeBody(this[kState].body)) {
+          if (!isUint8Array(chunk)) {
+            throw new TypeError('Expected Uint8Array chunk')
+          }
+
+          buffer.set(chunk, offset)
+          offset += chunk.length
+        }
+
+        return buffer.buffer
+      }
+
+      // if we don't have content length, then we have to allocate 2x the
+      // size of the body, once for consumed data, and once for the final buffer
+
+      // This could be optimized by using growable ArrayBuffer, but it's not
+      // implemented yet. https://github.com/tc39/proposal-resizablearraybuffer
+
+      const chunks = []
+      let size = 0
+
+      for await (const chunk of consumeBody(this[kState].body)) {
+        if (!isUint8Array(chunk)) {
+          throw new TypeError('Expected Uint8Array chunk')
+        }
+
+        chunks.push(chunk)
+        size += chunk.byteLength
+      }
+
+      const buffer = new Uint8Array(size)
+      let offset = 0
+
+      for (const chunk of chunks) {
+        buffer.set(chunk, offset)
+        offset += chunk.byteLength
+      }
+
+      return buffer.buffer
     },
 
     async text () {
@@ -312,8 +363,21 @@ function bodyMixinMethods (instance) {
         throw new TypeError('Illegal invocation')
       }
 
-      const blob = await this.blob()
-      return toUSVString(await blob.text())
+      let result = ''
+      const textDecoder = new TextDecoder()
+
+      for await (const chunk of consumeBody(this[kState].body)) {
+        if (!isUint8Array(chunk)) {
+          throw new TypeError('Expected Uint8Array chunk')
+        }
+
+        result += textDecoder.decode(chunk, { stream: true })
+      }
+
+      // flush
+      result += textDecoder.decode()
+
+      return result
     },
 
     async json () {
@@ -340,7 +404,18 @@ function bodyMixinMethods (instance) {
         // 1. Let entries be the result of parsing bytes.
         let entries
         try {
-          entries = new URLSearchParams(await this.text())
+          let text = ''
+          // application/x-www-form-urlencoded parser will keep the BOM.
+          // https://url.spec.whatwg.org/#concept-urlencoded-parser
+          const textDecoder = new TextDecoder('utf-8', { ignoreBOM: true })
+          for await (const chunk of consumeBody(this[kState].body)) {
+            if (!isUint8Array(chunk)) {
+              throw new TypeError('Expected Uint8Array chunk')
+            }
+            text += textDecoder.decode(chunk, { stream: true })
+          }
+          text += textDecoder.decode()
+          entries = new URLSearchParams(text)
         } catch (err) {
           // istanbul ignore next: Unclear when new URLSearchParams can fail on a string.
           // 2. If entries is failure, then throw a TypeError.

package/lib/fetch/constants.js

@@ -63,7 +63,7 @@ const subresource = [
 /** @type {globalThis['DOMException']} */
 const DOMException = globalThis.DOMException ?? (() => {
   // DOMException was only made a global in Node v17.0.0,
-  // but fetch supports >= v16.5.
+  // but fetch supports >= v16.8.
   try {
     atob('~')
   } catch (err) {

package/lib/fetch/dataURL.js

@@ -255,7 +255,7 @@ function percentDecode (input) {
   }
 
   // 3. Return output.
-  return Uint8Array.of(...output)
+  return Uint8Array.from(output)
 }
 
 // https://mimesniff.spec.whatwg.org/#parse-a-mime-type

package/lib/fetch/headers.js

@@ -110,6 +110,7 @@ class HeadersList {
   // https://fetch.spec.whatwg.org/#concept-header-list-set
   set (name, value) {
     this[kHeadersSortedMap] = null
+    name = name.toLowerCase()
 
     // 1. If list contains name, then set the value of
     //    the first such header to value and remove the

package/lib/fetch/index.js

@@ -33,7 +33,8 @@ const {
   isBlobLike,
   sameOrigin,
   isCancelled,
-  isAborted
+  isAborted,
+  isErrorLike
 } = require('./util')
 const { kState, kHeaders, kGuard, kRealm } = require('./symbols')
 const assert = require('assert')
@@ -1854,7 +1855,7 @@ async function httpNetworkFetch (
       timingInfo.decodedBodySize += bytes?.byteLength ?? 0
 
       // 6. If bytes is failure, then terminate fetchParams’s controller.
-      if (bytes instanceof Error) {
+      if (isErrorLike(bytes)) {
         fetchParams.controller.terminate(bytes)
         return
       }
@@ -1894,7 +1895,7 @@ async function httpNetworkFetch (
       // 3. Otherwise, if stream is readable, error stream with a TypeError.
       if (isReadable(stream)) {
         fetchParams.controller.controller.error(new TypeError('terminated', {
-          cause: reason instanceof Error ? reason : undefined
+          cause: isErrorLike(reason) ? reason : undefined
         }))
       }
     }
@@ -1942,14 +1943,17 @@ async function httpNetworkFetch (
           }
 
           let codings = []
+          let location = ''
 
           const headers = new Headers()
           for (let n = 0; n < headersList.length; n += 2) {
-            const key = headersList[n + 0].toString()
-            const val = headersList[n + 1].toString()
+            const key = headersList[n + 0].toString('latin1')
+            const val = headersList[n + 1].toString('latin1')
 
             if (key.toLowerCase() === 'content-encoding') {
               codings = val.split(',').map((x) => x.trim())
+            } else if (key.toLowerCase() === 'location') {
+              location = val
             }
 
             headers.append(key, val)
@@ -1960,7 +1964,7 @@ async function httpNetworkFetch (
           const decoders = []
 
           // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding
-          if (request.method !== 'HEAD' && request.method !== 'CONNECT' && !nullBodyStatus.includes(status)) {
+          if (request.method !== 'HEAD' && request.method !== 'CONNECT' && !nullBodyStatus.includes(status) && !(request.redirect === 'follow' && location)) {
             for (const coding of codings) {
               if (/(x-)?gzip/.test(coding)) {
                 decoders.push(zlib.createGunzip())
@@ -1980,7 +1984,7 @@ async function httpNetworkFetch (
             statusText,
             headersList: headers[kHeadersList],
             body: decoders.length
-              ? pipeline(this.body, ...decoders, () => {})
+              ? pipeline(this.body, ...decoders, () => { })
               : this.body.on('error', () => {})
           })