undici 7.9.0 → 7.11.0

package/lib/core/connect.js

@@ -12,64 +12,34 @@ let tls // include tls conditionally since it is not always available
 // resolve the same servername multiple times even when
 // re-use is enabled.
 
-let SessionCache
-// FIXME: remove workaround when the Node bug is fixed
-// https://github.com/nodejs/node/issues/49344#issuecomment-1741776308
-if (global.FinalizationRegistry && !(process.env.NODE_V8_COVERAGE || process.env.UNDICI_NO_FG)) {
-  SessionCache = class WeakSessionCache {
-    constructor (maxCachedSessions) {
-      this._maxCachedSessions = maxCachedSessions
-      this._sessionCache = new Map()
-      this._sessionRegistry = new global.FinalizationRegistry((key) => {
-        if (this._sessionCache.size < this._maxCachedSessions) {
-          return
-        }
-
-        const ref = this._sessionCache.get(key)
-        if (ref !== undefined && ref.deref() === undefined) {
-          this._sessionCache.delete(key)
-        }
-      })
-    }
-
-    get (sessionKey) {
-      const ref = this._sessionCache.get(sessionKey)
-      return ref ? ref.deref() : null
-    }
-
-    set (sessionKey, session) {
-      if (this._maxCachedSessions === 0) {
+const SessionCache = class WeakSessionCache {
+  constructor (maxCachedSessions) {
+    this._maxCachedSessions = maxCachedSessions
+    this._sessionCache = new Map()
+    this._sessionRegistry = new FinalizationRegistry((key) => {
+      if (this._sessionCache.size < this._maxCachedSessions) {
         return
       }
 
-      this._sessionCache.set(sessionKey, new WeakRef(session))
-      this._sessionRegistry.register(session, sessionKey)
-    }
-  }
-} else {
-  SessionCache = class SimpleSessionCache {
-    constructor (maxCachedSessions) {
-      this._maxCachedSessions = maxCachedSessions
-      this._sessionCache = new Map()
-    }
-
-    get (sessionKey) {
-      return this._sessionCache.get(sessionKey)
-    }
-
-    set (sessionKey, session) {
-      if (this._maxCachedSessions === 0) {
-        return
+      const ref = this._sessionCache.get(key)
+      if (ref !== undefined && ref.deref() === undefined) {
+        this._sessionCache.delete(key)
       }
+    })
+  }
 
-      if (this._sessionCache.size >= this._maxCachedSessions) {
-        // remove the oldest session
-        const { value: oldestKey } = this._sessionCache.keys().next()
-        this._sessionCache.delete(oldestKey)
-      }
+  get (sessionKey) {
+    const ref = this._sessionCache.get(sessionKey)
+    return ref ? ref.deref() : null
+  }
 
-      this._sessionCache.set(sessionKey, session)
+  set (sessionKey, session) {
+    if (this._maxCachedSessions === 0) {
+      return
     }
+
+    this._sessionCache.set(sessionKey, new WeakRef(session))
+    this._sessionRegistry.register(session, sessionKey)
   }
 }
 

package/lib/core/diagnostics.js

@@ -16,6 +16,8 @@ const channels = {
   // Request
   create: diagnosticsChannel.channel('undici:request:create'),
   bodySent: diagnosticsChannel.channel('undici:request:bodySent'),
+  bodyChunkSent: diagnosticsChannel.channel('undici:request:bodyChunkSent'),
+  bodyChunkReceived: diagnosticsChannel.channel('undici:request:bodyChunkReceived'),
   headers: diagnosticsChannel.channel('undici:request:headers'),
   trailers: diagnosticsChannel.channel('undici:request:trailers'),
   error: diagnosticsChannel.channel('undici:request:error'),
@@ -85,7 +87,7 @@ function trackClientEvents (debugLog = undiciDebugLog) {
       const {
         request: { method, path, origin }
       } = evt
-      debugLog('sending request to %s %s/%s', method, origin, path)
+      debugLog('sending request to %s %s%s', method, origin, path)
     })
 }
 
@@ -105,7 +107,7 @@ function trackRequestEvents (debugLog = undiciDebugLog) {
         response: { statusCode }
       } = evt
       debugLog(
-        'received response to %s %s/%s - HTTP %d',
+        'received response to %s %s%s - HTTP %d',
         method,
         origin,
         path,
@@ -118,7 +120,7 @@ function trackRequestEvents (debugLog = undiciDebugLog) {
       const {
         request: { method, path, origin }
       } = evt
-      debugLog('trailers received from %s %s/%s', method, origin, path)
+      debugLog('trailers received from %s %s%s', method, origin, path)
     })
 
   diagnosticsChannel.subscribe('undici:request:error',
@@ -128,7 +130,7 @@ function trackRequestEvents (debugLog = undiciDebugLog) {
         error
       } = evt
       debugLog(
-        'request to %s %s/%s errored - %s',
+        'request to %s %s%s errored - %s',
         method,
         origin,
         path,

package/lib/core/request.js

@@ -194,6 +194,9 @@ class Request {
   }
 
   onBodySent (chunk) {
+    if (channels.bodyChunkSent.hasSubscribers) {
+      channels.bodyChunkSent.publish({ request: this, chunk })
+    }
     if (this[kHandler].onBodySent) {
       try {
         return this[kHandler].onBodySent(chunk)
@@ -252,6 +255,9 @@ class Request {
     assert(!this.aborted)
     assert(!this.completed)
 
+    if (channels.bodyChunkReceived.hasSubscribers) {
+      channels.bodyChunkReceived.publish({ request: this, chunk })
+    }
     try {
       return this[kHandler].onData(chunk)
     } catch (err) {

package/lib/core/util.js

@@ -6,7 +6,6 @@ const { IncomingMessage } = require('node:http')
 const stream = require('node:stream')
 const net = require('node:net')
 const { Blob } = require('node:buffer')
-const nodeUtil = require('node:util')
 const { stringify } = require('node:querystring')
 const { EventEmitter: EE } = require('node:events')
 const timers = require('../util/timers')
@@ -660,48 +659,6 @@ function addAbortListener (signal, listener) {
   return () => signal.removeListener('abort', listener)
 }
 
-/**
- * @function
- * @param {string} value
- * @returns {string}
- */
-const toUSVString = (() => {
-  if (typeof String.prototype.toWellFormed === 'function') {
-    /**
-     * @param {string} value
-     * @returns {string}
-     */
-    return (value) => `${value}`.toWellFormed()
-  } else {
-    /**
-     * @param {string} value
-     * @returns {string}
-     */
-    return nodeUtil.toUSVString
-  }
-})()
-
-/**
- * @param {*} value
- * @returns {boolean}
- */
-// TODO: move this to webidl
-const isUSVString = (() => {
-  if (typeof String.prototype.isWellFormed === 'function') {
-    /**
-     * @param {*} value
-     * @returns {boolean}
-     */
-    return (value) => `${value}`.isWellFormed()
-  } else {
-    /**
-     * @param {*} value
-     * @returns {boolean}
-     */
-    return (value) => toUSVString(value) === `${value}`
-  }
-})()
-
 /**
  * @see https://tools.ietf.org/html/rfc7230#section-3.2.6
  * @param {number} c
@@ -943,8 +900,6 @@ Object.setPrototypeOf(normalizedMethodRecords, null)
 module.exports = {
   kEnumerableProperty,
   isDisturbed,
-  toUSVString,
-  isUSVString,
   isBlobLike,
   parseOrigin,
   parseURL,

package/lib/dispatcher/agent.js

@@ -45,22 +45,35 @@ class Agent extends DispatcherBase {
     }
 
     this[kOnConnect] = (origin, targets) => {
+      const result = this[kClients].get(origin)
+      if (result) {
+        result.count += 1
+      }
       this.emit('connect', origin, [this, ...targets])
     }
 
     this[kOnDisconnect] = (origin, targets, err) => {
+      const result = this[kClients].get(origin)
+      if (result) {
+        result.count -= 1
+        if (result.count <= 0) {
+          this[kClients].delete(origin)
+          result.dispatcher.destroy()
+        }
+      }
       this.emit('disconnect', origin, [this, ...targets], err)
     }
 
     this[kOnConnectionError] = (origin, targets, err) => {
+      // TODO: should this decrement result.count here?
       this.emit('connectionError', origin, [this, ...targets], err)
     }
   }
 
   get [kRunning] () {
     let ret = 0
-    for (const client of this[kClients].values()) {
-      ret += client[kRunning]
+    for (const { dispatcher } of this[kClients].values()) {
+      ret += dispatcher[kRunning]
     }
     return ret
   }
@@ -73,8 +86,8 @@ class Agent extends DispatcherBase {
       throw new InvalidArgumentError('opts.origin must be a non-empty string or URL.')
     }
 
-    let dispatcher = this[kClients].get(key)
-
+    const result = this[kClients].get(key)
+    let dispatcher = result && result.dispatcher
     if (!dispatcher) {
       dispatcher = this[kFactory](opts.origin, this[kOptions])
         .on('drain', this[kOnDrain])
@@ -82,10 +95,7 @@ class Agent extends DispatcherBase {
         .on('disconnect', this[kOnDisconnect])
         .on('connectionError', this[kOnConnectionError])
 
-      // This introduces a tiny memory leak, as dispatchers are never removed from the map.
-      // TODO(mcollina): remove te timer when the client/pool do not have any more
-      // active connections.
-      this[kClients].set(key, dispatcher)
+      this[kClients].set(key, { count: 0, dispatcher })
     }
 
     return dispatcher.dispatch(opts, handler)
@@ -93,8 +103,8 @@ class Agent extends DispatcherBase {
 
   async [kClose] () {
     const closePromises = []
-    for (const client of this[kClients].values()) {
-      closePromises.push(client.close())
+    for (const { dispatcher } of this[kClients].values()) {
+      closePromises.push(dispatcher.close())
     }
     this[kClients].clear()
 
@@ -103,8 +113,8 @@ class Agent extends DispatcherBase {
 
   async [kDestroy] (err) {
     const destroyPromises = []
-    for (const client of this[kClients].values()) {
-      destroyPromises.push(client.destroy(err))
+    for (const { dispatcher } of this[kClients].values()) {
+      destroyPromises.push(dispatcher.destroy(err))
     }
     this[kClients].clear()
 
@@ -113,9 +123,9 @@ class Agent extends DispatcherBase {
 
   get stats () {
     const allClientStats = {}
-    for (const client of this[kClients].values()) {
-      if (client.stats) {
-        allClientStats[client[kUrl].origin] = client.stats
+    for (const { dispatcher } of this[kClients].values()) {
+      if (dispatcher.stats) {
+        allClientStats[dispatcher[kUrl].origin] = dispatcher.stats
       }
     }
     return allClientStats

package/lib/dispatcher/client-h1.js

@@ -249,7 +249,7 @@ class Parser {
           this.timeout = timers.setFastTimeout(onParserTimeout, delay, new WeakRef(this))
         } else {
           this.timeout = setTimeout(onParserTimeout, delay, new WeakRef(this))
-          this.timeout.unref()
+          this.timeout?.unref()
         }
       }
 

package/lib/dispatcher/pool.js

@@ -5,7 +5,8 @@ const {
   kClients,
   kNeedDrain,
   kAddClient,
-  kGetDispatcher
+  kGetDispatcher,
+  kRemoveClient
 } = require('./pool-base')
 const Client = require('./client')
 const {
@@ -35,6 +36,7 @@ class Pool extends PoolBase {
     autoSelectFamily,
     autoSelectFamilyAttemptTimeout,
     allowH2,
+    clientTtl,
     ...options
   } = {}) {
     if (connections != null && (!Number.isFinite(connections) || connections < 0)) {
@@ -65,12 +67,20 @@ class Pool extends PoolBase {
 
     this[kConnections] = connections || null
     this[kUrl] = util.parseOrigin(origin)
-    this[kOptions] = { ...util.deepClone(options), connect, allowH2 }
+    this[kOptions] = { ...util.deepClone(options), connect, allowH2, clientTtl }
     this[kOptions].interceptors = options.interceptors
       ? { ...options.interceptors }
       : undefined
     this[kFactory] = factory
 
+    this.on('connect', (origin, targets) => {
+      if (clientTtl != null && clientTtl > 0) {
+        for (const target of targets) {
+          Object.assign(target, { ttl: Date.now() })
+        }
+      }
+    })
+
     this.on('connectionError', (origin, targets, error) => {
       // If a connection error occurs, we remove the client from the pool,
       // and emit a connectionError event. They will not be re-used.
@@ -87,8 +97,12 @@ class Pool extends PoolBase {
   }
 
   [kGetDispatcher] () {
+    const clientTtlOption = this[kOptions].clientTtl
     for (const client of this[kClients]) {
-      if (!client[kNeedDrain]) {
+      // check ttl of client and if it's stale, remove it from the pool
+      if (clientTtlOption != null && clientTtlOption > 0 && client.ttl && ((Date.now() - client.ttl) > clientTtlOption)) {
+        this[kRemoveClient](client)
+      } else if (!client[kNeedDrain]) {
         return client
       }
     }

package/lib/dispatcher/proxy-agent.js

@@ -1,12 +1,13 @@
 'use strict'
 
-const { kProxy, kClose, kDestroy } = require('../core/symbols')
+const { kProxy, kClose, kDestroy, kDispatch, kConnector } = require('../core/symbols')
 const { URL } = require('node:url')
 const Agent = require('./agent')
 const Pool = require('./pool')
 const DispatcherBase = require('./dispatcher-base')
 const { InvalidArgumentError, RequestAbortedError, SecureProxyConnectionError } = require('../core/errors')
 const buildConnector = require('../core/connect')
+const Client = require('./client')
 
 const kAgent = Symbol('proxy agent')
 const kClient = Symbol('proxy client')
@@ -14,6 +15,7 @@ const kProxyHeaders = Symbol('proxy headers')
 const kRequestTls = Symbol('request tls settings')
 const kProxyTls = Symbol('proxy tls settings')
 const kConnectEndpoint = Symbol('connect endpoint function')
+const kTunnelProxy = Symbol('tunnel proxy')
 
 function defaultProtocolPort (protocol) {
   return protocol === 'https:' ? 443 : 80
@@ -25,6 +27,61 @@ function defaultFactory (origin, opts) {
 
 const noop = () => {}
 
+class ProxyClient extends DispatcherBase {
+  #client = null
+  constructor (origin, opts) {
+    if (typeof origin === 'string') {
+      origin = new URL(origin)
+    }
+
+    if (origin.protocol !== 'http:' && origin.protocol !== 'https:') {
+      throw new InvalidArgumentError('ProxyClient only supports http and https protocols')
+    }
+
+    super()
+
+    this.#client = new Client(origin, opts)
+  }
+
+  async [kClose] () {
+    await this.#client.close()
+  }
+
+  async [kDestroy] () {
+    await this.#client.destroy()
+  }
+
+  async [kDispatch] (opts, handler) {
+    const { method, origin } = opts
+    if (method === 'CONNECT') {
+      this.#client[kConnector]({
+        origin,
+        port: opts.port || defaultProtocolPort(opts.protocol),
+        path: opts.host,
+        signal: opts.signal,
+        headers: {
+          ...this[kProxyHeaders],
+          host: opts.host
+        },
+        servername: this[kProxyTls]?.servername || opts.servername
+      },
+      (err, socket) => {
+        if (err) {
+          handler.callback(err)
+        } else {
+          handler.callback(null, { socket, statusCode: 200 })
+        }
+      }
+      )
+      return
+    }
+    if (typeof origin === 'string') {
+      opts.origin = new URL(origin)
+    }
+
+    return this.#client.dispatch(opts, handler)
+  }
+}
 class ProxyAgent extends DispatcherBase {
   constructor (opts) {
     if (!opts || (typeof opts === 'object' && !(opts instanceof URL) && !opts.uri)) {
@@ -36,6 +93,8 @@ class ProxyAgent extends DispatcherBase {
       throw new InvalidArgumentError('Proxy opts.clientFactory must be a function.')
     }
 
+    const { proxyTunnel = true } = opts
+
     super()
 
     const url = this.#getUrl(opts)
@@ -57,9 +116,19 @@ class ProxyAgent extends DispatcherBase {
       this[kProxyHeaders]['proxy-authorization'] = `Basic ${Buffer.from(`${decodeURIComponent(username)}:${decodeURIComponent(password)}`).toString('base64')}`
     }
 
+    const factory = (!proxyTunnel && protocol === 'http:')
+      ? (origin, options) => {
+          if (origin.protocol === 'http:') {
+            return new ProxyClient(origin, options)
+          }
+          return new Client(origin, options)
+        }
+      : undefined
+
     const connect = buildConnector({ ...opts.proxyTls })
     this[kConnectEndpoint] = buildConnector({ ...opts.requestTls })
-    this[kClient] = clientFactory(url, { connect })
+    this[kClient] = clientFactory(url, { connect, factory })
+    this[kTunnelProxy] = proxyTunnel
     this[kAgent] = new Agent({
       ...opts,
       connect: async (opts, callback) => {
@@ -75,7 +144,8 @@ class ProxyAgent extends DispatcherBase {
             signal: opts.signal,
             headers: {
               ...this[kProxyHeaders],
-              host: opts.host
+              host: opts.host,
+              ...(opts.connections == null || opts.connections > 0 ? { 'proxy-connection': 'keep-alive' } : {})
             },
             servername: this[kProxyTls]?.servername || proxyHostname
           })
@@ -115,6 +185,10 @@ class ProxyAgent extends DispatcherBase {
       headers.host = host
     }
 
+    if (!this.#shouldConnect(new URL(opts.origin))) {
+      opts.path = opts.origin + opts.path
+    }
+
     return this[kAgent].dispatch(
       {
         ...opts,
@@ -147,6 +221,19 @@ class ProxyAgent extends DispatcherBase {
     await this[kAgent].destroy()
     await this[kClient].destroy()
   }
+
+  #shouldConnect (uri) {
+    if (typeof uri === 'string') {
+      uri = new URL(uri)
+    }
+    if (this[kTunnelProxy]) {
+      return true
+    }
+    if (uri.protocol !== 'http:' || this[kProxy].protocol !== 'http:') {
+      return true
+    }
+    return false
+  }
 }
 
 /**