undici 7.9.0 → 7.10.0

package/docs/docs/api/CacheStore.md

@@ -13,8 +13,28 @@ The `MemoryCacheStore` stores the responses in-memory.
 
 **Options**
 
+- `maxSize` - The maximum total size in bytes of all stored responses. Default `Infinity`.
 - `maxCount` - The maximum amount of responses to store. Default `Infinity`.
-- `maxEntrySize` - The maximum size in bytes that a response's body can be. If a response's body is greater than or equal to this, the response will not be cached.
+- `maxEntrySize` - The maximum size in bytes that a response's body can be. If a response's body is greater than or equal to this, the response will not be cached. Default `Infinity`.
+
+### Getters
+
+#### `MemoryCacheStore.size`
+
+Returns the current total size in bytes of all stored responses.
+
+### Methods
+
+#### `MemoryCacheStore.isFull()`
+
+Returns a boolean indicating whether the cache has reached its maximum size or count.
+
+### Events
+
+#### `'maxSizeExceeded'`
+
+Emitted when the cache exceeds its maximum size or count limits. The event payload contains `size`, `maxSize`, `count`, and `maxCount` properties.
+
 
 ### `SqliteCacheStore`
 
@@ -26,7 +46,7 @@ The `SqliteCacheStore` is only exposed if the `node:sqlite` api is present.
 
 - `location` - The location of the SQLite database to use. Default `:memory:`.
 - `maxCount` - The maximum number of entries to store in the database. Default `Infinity`.
-- `maxEntrySize` - The maximum size in bytes that a resposne's body can be. If a response's body is greater than or equal to this, the response will not be cached. Default `Infinity`.
+- `maxEntrySize` - The maximum size in bytes that a response's body can be. If a response's body is greater than or equal to this, the response will not be cached. Default `Infinity`.
 
 ## Defining a Custom Cache Store
 

package/docs/docs/api/Pool.md

@@ -19,6 +19,7 @@ Extends: [`ClientOptions`](/docs/docs/api/Client.md#parameter-clientoptions)
 
 * **factory** `(origin: URL, opts: Object) => Dispatcher` - Default: `(origin, opts) => new Client(origin, opts)`
 * **connections** `number | null` (optional) - Default: `null` - The number of `Client` instances to create. When set to `null`, the `Pool` instance will create an unlimited amount of `Client` instances.
+* **clientTtl** `number | null` (optional) - Default: `null` - The amount of time before a `Client` instance is removed from the `Pool` and closed.   When set to `null`, `Client` instances will not be removed or closed based on age.
 
 ## Instance Properties
 

package/docs/docs/api/ProxyAgent.md

@@ -25,6 +25,7 @@ For detailed information on the parsing process and potential validation errors,
 * **clientFactory** `(origin: URL, opts: Object) => Dispatcher` (optional) - Default: `(origin, opts) => new Pool(origin, opts)`
 * **requestTls** `BuildOptions` (optional) - Options object passed when creating the underlying socket via the connector builder for the request. It extends from [`Client#ConnectOptions`](/docs/docs/api/Client.md#parameter-connectoptions).
 * **proxyTls** `BuildOptions` (optional) - Options object passed when creating the underlying socket via the connector builder for the proxy server. It extends from [`Client#ConnectOptions`](/docs/docs/api/Client.md#parameter-connectoptions).
+* **proxyTunnel** `boolean` (optional) - By default, ProxyAgent will request that the Proxy facilitate a tunnel between the endpoint and the agent. Setting `proxyTunnel` to false avoids issuing a CONNECT extension, and includes the endpoint domain and path in each request.
 
 Examples:
 

package/lib/cache/memory-cache-store.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const { Writable } = require('node:stream')
+const { EventEmitter } = require('node:events')
 const { assertCacheKey, assertCacheValue } = require('../util/cache.js')
 
 /**
@@ -12,8 +13,9 @@ const { assertCacheKey, assertCacheValue } = require('../util/cache.js')
 
 /**
  * @implements {CacheStore}
+ * @extends {EventEmitter}
  */
-class MemoryCacheStore {
+class MemoryCacheStore extends EventEmitter {
   #maxCount = Infinity
   #maxSize = Infinity
   #maxEntrySize = Infinity
@@ -21,11 +23,13 @@ class MemoryCacheStore {
   #size = 0
   #count = 0
   #entries = new Map()
+  #hasEmittedMaxSizeEvent = false
 
   /**
    * @param {import('../../types/cache-interceptor.d.ts').default.MemoryCacheStoreOpts | undefined} [opts]
    */
   constructor (opts) {
+    super()
     if (opts) {
       if (typeof opts !== 'object') {
         throw new TypeError('MemoryCacheStore options must be an object')
@@ -66,6 +70,22 @@ class MemoryCacheStore {
     }
   }
 
+  /**
+   * Get the current size of the cache in bytes
+   * @returns {number} The current size of the cache in bytes
+   */
+  get size () {
+    return this.#size
+  }
+
+  /**
+   * Check if the cache is full (either max size or max count reached)
+   * @returns {boolean} True if the cache is full, false otherwise
+   */
+  isFull () {
+    return this.#size >= this.#maxSize || this.#count >= this.#maxCount
+  }
+
   /**
    * @param {import('../../types/cache-interceptor.d.ts').default.CacheKey} req
    * @returns {import('../../types/cache-interceptor.d.ts').default.GetResult | undefined}
@@ -144,7 +164,20 @@ class MemoryCacheStore {
 
         store.#size += entry.size
 
+        // Check if cache is full and emit event if needed
         if (store.#size > store.#maxSize || store.#count > store.#maxCount) {
+          // Emit maxSizeExceeded event if we haven't already
+          if (!store.#hasEmittedMaxSizeEvent) {
+            store.emit('maxSizeExceeded', {
+              size: store.#size,
+              maxSize: store.#maxSize,
+              count: store.#count,
+              maxCount: store.#maxCount
+            })
+            store.#hasEmittedMaxSizeEvent = true
+          }
+
+          // Perform eviction
           for (const [key, entries] of store.#entries) {
             for (const entry of entries.splice(0, entries.length / 2)) {
               store.#size -= entry.size
@@ -154,6 +187,11 @@ class MemoryCacheStore {
               store.#entries.delete(key)
             }
           }
+
+          // Reset the event flag after eviction
+          if (store.#size < store.#maxSize && store.#count < store.#maxCount) {
+            store.#hasEmittedMaxSizeEvent = false
+          }
         }
 
         callback(null)

package/lib/dispatcher/agent.js

@@ -45,22 +45,35 @@ class Agent extends DispatcherBase {
     }
 
     this[kOnConnect] = (origin, targets) => {
+      const result = this[kClients].get(origin)
+      if (result) {
+        result.count += 1
+      }
       this.emit('connect', origin, [this, ...targets])
     }
 
     this[kOnDisconnect] = (origin, targets, err) => {
+      const result = this[kClients].get(origin)
+      if (result) {
+        result.count -= 1
+        if (result.count <= 0) {
+          this[kClients].delete(origin)
+          result.dispatcher.destroy()
+        }
+      }
       this.emit('disconnect', origin, [this, ...targets], err)
     }
 
     this[kOnConnectionError] = (origin, targets, err) => {
+      // TODO: should this decrement result.count here?
       this.emit('connectionError', origin, [this, ...targets], err)
     }
   }
 
   get [kRunning] () {
     let ret = 0
-    for (const client of this[kClients].values()) {
-      ret += client[kRunning]
+    for (const { dispatcher } of this[kClients].values()) {
+      ret += dispatcher[kRunning]
     }
     return ret
   }
@@ -73,8 +86,8 @@ class Agent extends DispatcherBase {
       throw new InvalidArgumentError('opts.origin must be a non-empty string or URL.')
     }
 
-    let dispatcher = this[kClients].get(key)
-
+    const result = this[kClients].get(key)
+    let dispatcher = result && result.dispatcher
     if (!dispatcher) {
       dispatcher = this[kFactory](opts.origin, this[kOptions])
         .on('drain', this[kOnDrain])
@@ -82,10 +95,7 @@ class Agent extends DispatcherBase {
         .on('disconnect', this[kOnDisconnect])
         .on('connectionError', this[kOnConnectionError])
 
-      // This introduces a tiny memory leak, as dispatchers are never removed from the map.
-      // TODO(mcollina): remove te timer when the client/pool do not have any more
-      // active connections.
-      this[kClients].set(key, dispatcher)
+      this[kClients].set(key, { count: 0, dispatcher })
     }
 
     return dispatcher.dispatch(opts, handler)
@@ -93,8 +103,8 @@ class Agent extends DispatcherBase {
 
   async [kClose] () {
     const closePromises = []
-    for (const client of this[kClients].values()) {
-      closePromises.push(client.close())
+    for (const { dispatcher } of this[kClients].values()) {
+      closePromises.push(dispatcher.close())
     }
     this[kClients].clear()
 
@@ -103,8 +113,8 @@ class Agent extends DispatcherBase {
 
   async [kDestroy] (err) {
     const destroyPromises = []
-    for (const client of this[kClients].values()) {
-      destroyPromises.push(client.destroy(err))
+    for (const { dispatcher } of this[kClients].values()) {
+      destroyPromises.push(dispatcher.destroy(err))
     }
     this[kClients].clear()
 
@@ -113,9 +123,9 @@ class Agent extends DispatcherBase {
 
   get stats () {
     const allClientStats = {}
-    for (const client of this[kClients].values()) {
-      if (client.stats) {
-        allClientStats[client[kUrl].origin] = client.stats
+    for (const { dispatcher } of this[kClients].values()) {
+      if (dispatcher.stats) {
+        allClientStats[dispatcher[kUrl].origin] = dispatcher.stats
       }
     }
     return allClientStats

package/lib/dispatcher/pool.js

@@ -5,7 +5,8 @@ const {
   kClients,
   kNeedDrain,
   kAddClient,
-  kGetDispatcher
+  kGetDispatcher,
+  kRemoveClient
 } = require('./pool-base')
 const Client = require('./client')
 const {
@@ -35,6 +36,7 @@ class Pool extends PoolBase {
     autoSelectFamily,
     autoSelectFamilyAttemptTimeout,
     allowH2,
+    clientTtl,
     ...options
   } = {}) {
     if (connections != null && (!Number.isFinite(connections) || connections < 0)) {
@@ -65,12 +67,20 @@ class Pool extends PoolBase {
 
     this[kConnections] = connections || null
     this[kUrl] = util.parseOrigin(origin)
-    this[kOptions] = { ...util.deepClone(options), connect, allowH2 }
+    this[kOptions] = { ...util.deepClone(options), connect, allowH2, clientTtl }
     this[kOptions].interceptors = options.interceptors
       ? { ...options.interceptors }
       : undefined
     this[kFactory] = factory
 
+    this.on('connect', (origin, targets) => {
+      if (clientTtl != null && clientTtl > 0) {
+        for (const target of targets) {
+          Object.assign(target, { ttl: Date.now() })
+        }
+      }
+    })
+
     this.on('connectionError', (origin, targets, error) => {
       // If a connection error occurs, we remove the client from the pool,
       // and emit a connectionError event. They will not be re-used.
@@ -87,8 +97,12 @@ class Pool extends PoolBase {
   }
 
   [kGetDispatcher] () {
+    const clientTtlOption = this[kOptions].clientTtl
     for (const client of this[kClients]) {
-      if (!client[kNeedDrain]) {
+      // check ttl of client and if it's stale, remove it from the pool
+      if (clientTtlOption != null && clientTtlOption > 0 && client.ttl && ((Date.now() - client.ttl) > clientTtlOption)) {
+        this[kRemoveClient](client)
+      } else if (!client[kNeedDrain]) {
         return client
       }
     }

package/lib/dispatcher/proxy-agent.js

@@ -1,12 +1,13 @@
 'use strict'
 
-const { kProxy, kClose, kDestroy } = require('../core/symbols')
+const { kProxy, kClose, kDestroy, kDispatch, kConnector } = require('../core/symbols')
 const { URL } = require('node:url')
 const Agent = require('./agent')
 const Pool = require('./pool')
 const DispatcherBase = require('./dispatcher-base')
 const { InvalidArgumentError, RequestAbortedError, SecureProxyConnectionError } = require('../core/errors')
 const buildConnector = require('../core/connect')
+const Client = require('./client')
 
 const kAgent = Symbol('proxy agent')
 const kClient = Symbol('proxy client')
@@ -14,6 +15,7 @@ const kProxyHeaders = Symbol('proxy headers')
 const kRequestTls = Symbol('request tls settings')
 const kProxyTls = Symbol('proxy tls settings')
 const kConnectEndpoint = Symbol('connect endpoint function')
+const kTunnelProxy = Symbol('tunnel proxy')
 
 function defaultProtocolPort (protocol) {
   return protocol === 'https:' ? 443 : 80
@@ -25,6 +27,61 @@ function defaultFactory (origin, opts) {
 
 const noop = () => {}
 
+class ProxyClient extends DispatcherBase {
+  #client = null
+  constructor (origin, opts) {
+    if (typeof origin === 'string') {
+      origin = new URL(origin)
+    }
+
+    if (origin.protocol !== 'http:' && origin.protocol !== 'https:') {
+      throw new InvalidArgumentError('ProxyClient only supports http and https protocols')
+    }
+
+    super()
+
+    this.#client = new Client(origin, opts)
+  }
+
+  async [kClose] () {
+    await this.#client.close()
+  }
+
+  async [kDestroy] () {
+    await this.#client.destroy()
+  }
+
+  async [kDispatch] (opts, handler) {
+    const { method, origin } = opts
+    if (method === 'CONNECT') {
+      this.#client[kConnector]({
+        origin,
+        port: opts.port || defaultProtocolPort(opts.protocol),
+        path: opts.host,
+        signal: opts.signal,
+        headers: {
+          ...this[kProxyHeaders],
+          host: opts.host
+        },
+        servername: this[kProxyTls]?.servername || opts.servername
+      },
+      (err, socket) => {
+        if (err) {
+          handler.callback(err)
+        } else {
+          handler.callback(null, { socket, statusCode: 200 })
+        }
+      }
+      )
+      return
+    }
+    if (typeof origin === 'string') {
+      opts.origin = new URL(origin)
+    }
+
+    return this.#client.dispatch(opts, handler)
+  }
+}
 class ProxyAgent extends DispatcherBase {
   constructor (opts) {
     if (!opts || (typeof opts === 'object' && !(opts instanceof URL) && !opts.uri)) {
@@ -36,6 +93,8 @@ class ProxyAgent extends DispatcherBase {
       throw new InvalidArgumentError('Proxy opts.clientFactory must be a function.')
     }
 
+    const { proxyTunnel = true } = opts
+
     super()
 
     const url = this.#getUrl(opts)
@@ -57,9 +116,19 @@ class ProxyAgent extends DispatcherBase {
       this[kProxyHeaders]['proxy-authorization'] = `Basic ${Buffer.from(`${decodeURIComponent(username)}:${decodeURIComponent(password)}`).toString('base64')}`
     }
 
+    const factory = (!proxyTunnel && protocol === 'http:')
+      ? (origin, options) => {
+          if (origin.protocol === 'http:') {
+            return new ProxyClient(origin, options)
+          }
+          return new Client(origin, options)
+        }
+      : undefined
+
     const connect = buildConnector({ ...opts.proxyTls })
     this[kConnectEndpoint] = buildConnector({ ...opts.requestTls })
-    this[kClient] = clientFactory(url, { connect })
+    this[kClient] = clientFactory(url, { connect, factory })
+    this[kTunnelProxy] = proxyTunnel
     this[kAgent] = new Agent({
       ...opts,
       connect: async (opts, callback) => {
@@ -115,6 +184,10 @@ class ProxyAgent extends DispatcherBase {
       headers.host = host
     }
 
+    if (!this.#shouldConnect(new URL(opts.origin))) {
+      opts.path = opts.origin + opts.path
+    }
+
     return this[kAgent].dispatch(
       {
         ...opts,
@@ -147,6 +220,19 @@ class ProxyAgent extends DispatcherBase {
     await this[kAgent].destroy()
     await this[kClient].destroy()
   }
+
+  #shouldConnect (uri) {
+    if (typeof uri === 'string') {
+      uri = new URL(uri)
+    }
+    if (this[kTunnelProxy]) {
+      return true
+    }
+    if (uri.protocol !== 'http:' || this[kProxy].protocol !== 'http:') {
+      return true
+    }
+    return false
+  }
 }
 
 /**

package/lib/mock/mock-agent.js

@@ -159,7 +159,7 @@ class MockAgent extends Dispatcher {
   }
 
   [kMockAgentSet] (origin, dispatcher) {
-    this[kClients].set(origin, dispatcher)
+    this[kClients].set(origin, { count: 0, dispatcher })
   }
 
   [kFactory] (origin) {
@@ -171,9 +171,9 @@ class MockAgent extends Dispatcher {
 
   [kMockAgentGet] (origin) {
     // First check if we can immediately find it
-    const client = this[kClients].get(origin)
-    if (client) {
-      return client
+    const result = this[kClients].get(origin)
+    if (result?.dispatcher) {
+      return result.dispatcher
     }
 
     // If the origin is not a string create a dummy parent pool and return to user
@@ -184,11 +184,11 @@ class MockAgent extends Dispatcher {
     }
 
     // If we match, create a pool and assign the same dispatches
-    for (const [keyMatcher, nonExplicitDispatcher] of Array.from(this[kClients])) {
-      if (nonExplicitDispatcher && typeof keyMatcher !== 'string' && matchValue(keyMatcher, origin)) {
+    for (const [keyMatcher, result] of Array.from(this[kClients])) {
+      if (result && typeof keyMatcher !== 'string' && matchValue(keyMatcher, origin)) {
         const dispatcher = this[kFactory](origin)
         this[kMockAgentSet](origin, dispatcher)
-        dispatcher[kDispatches] = nonExplicitDispatcher[kDispatches]
+        dispatcher[kDispatches] = result.dispatcher[kDispatches]
         return dispatcher
       }
     }
@@ -202,7 +202,7 @@ class MockAgent extends Dispatcher {
     const mockAgentClients = this[kClients]
 
     return Array.from(mockAgentClients.entries())
-      .flatMap(([origin, scope]) => scope[kDispatches].map(dispatch => ({ ...dispatch, origin })))
+      .flatMap(([origin, result]) => result.dispatcher[kDispatches].map(dispatch => ({ ...dispatch, origin })))
       .filter(({ pending }) => pending)
   }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "undici",
-  "version": "7.9.0",
+  "version": "7.10.0",
   "description": "An HTTP/1.1 client, written from scratch for Node.js",
   "homepage": "https://undici.nodejs.org",
   "bugs": {

package/types/pool.d.ts

@@ -33,6 +33,8 @@ declare namespace Pool {
     factory?(origin: URL, opts: object): Dispatcher;
     /** The max number of clients to create. `null` if no limit. Default `null`. */
     connections?: number | null;
+    /** The amount of time before a client is removed from the pool and closed. `null` if no time limit. Default `null` */
+    clientTtl?: number | null;
 
     interceptors?: { Pool?: readonly Dispatcher.DispatchInterceptor[] } & Client.Options['interceptors']
   }

package/types/proxy-agent.d.ts

@@ -24,5 +24,6 @@ declare namespace ProxyAgent {
     requestTls?: buildConnector.BuildOptions;
     proxyTls?: buildConnector.BuildOptions;
     clientFactory?(origin: URL, opts: object): Dispatcher;
+    proxyTunnel?: boolean;
   }
 }