undici 7.16.0 → 7.18.0

package/lib/interceptor/cache.js

@@ -9,6 +9,8 @@ const CacheRevalidationHandler = require('../handler/cache-revalidation-handler'
 const { assertCacheStore, assertCacheMethods, makeCacheKey, normalizeHeaders, parseCacheControlHeader } = require('../util/cache.js')
 const { AbortError } = require('../core/errors.js')
 
+const nop = () => {}
+
 /**
  * @typedef {(options: import('../../types/dispatcher.d.ts').default.DispatchOptions, handler: import('../../types/dispatcher.d.ts').default.DispatchHandler) => void} DispatchFn
  */
@@ -16,19 +18,34 @@ const { AbortError } = require('../core/errors.js')
 /**
  * @param {import('../../types/cache-interceptor.d.ts').default.GetResult} result
  * @param {import('../../types/cache-interceptor.d.ts').default.CacheControlDirectives | undefined} cacheControlDirectives
+ * @param {import('../../types/dispatcher.d.ts').default.RequestOptions} opts
  * @returns {boolean}
  */
-function needsRevalidation (result, cacheControlDirectives) {
+function needsRevalidation (result, cacheControlDirectives, { headers = {} }) {
+  // Always revalidate requests with the no-cache request directive.
   if (cacheControlDirectives?.['no-cache']) {
-    // Always revalidate requests with the no-cache request directive
     return true
   }
 
+  // Always revalidate requests with unqualified no-cache response directive.
   if (result.cacheControlDirectives?.['no-cache'] && !Array.isArray(result.cacheControlDirectives['no-cache'])) {
-    // Always revalidate requests with unqualified no-cache response directive
     return true
   }
 
+  // Always revalidate requests with conditional headers.
+  if (headers['if-modified-since'] || headers['if-none-match']) {
+    return true
+  }
+
+  return false
+}
+
+/**
+ * @param {import('../../types/cache-interceptor.d.ts').default.GetResult} result
+ * @param {import('../../types/cache-interceptor.d.ts').default.CacheControlDirectives | undefined} cacheControlDirectives
+ * @returns {boolean}
+ */
+function isStale (result, cacheControlDirectives) {
   const now = Date.now()
   if (now > result.staleAt) {
     // Response is stale
@@ -102,7 +119,7 @@ function handleUncachedResponse (
       }
 
       if (typeof handler.onHeaders === 'function') {
-        handler.onHeaders(504, [], () => {}, 'Gateway Timeout')
+        handler.onHeaders(504, [], nop, 'Gateway Timeout')
         if (aborted) {
           return
         }
@@ -239,8 +256,11 @@ function handleResult (
     return dispatch(opts, handler)
   }
 
+  const stale = isStale(result, reqCacheControl)
+  const revalidate = needsRevalidation(result, reqCacheControl, opts)
+
   // Check if the response is stale
-  if (needsRevalidation(result, reqCacheControl)) {
+  if (stale || revalidate) {
     if (util.isStream(opts.body) && util.bodyLength(opts.body) !== 0) {
       // If body is a stream we can't revalidate...
       // TODO (fix): This could be less strict...
@@ -248,8 +268,8 @@ function handleResult (
     }
 
     // RFC 5861: If we're within stale-while-revalidate window, serve stale immediately
-    // and revalidate in background
-    if (withinStaleWhileRevalidateWindow(result)) {
+    // and revalidate in background, unless immediate revalidation is necessary
+    if (!revalidate && withinStaleWhileRevalidateWindow(result)) {
       // Serve stale response immediately
       sendCachedValue(handler, opts, result, age, null, true)
 
@@ -323,9 +343,10 @@ function handleResult (
       new CacheRevalidationHandler(
         (success, context) => {
           if (success) {
-            sendCachedValue(handler, opts, result, age, context, true)
+            // TODO: successful revalidation should be considered fresh (not give stale warning).
+            sendCachedValue(handler, opts, result, age, context, stale)
           } else if (util.isStream(result.body)) {
-            result.body.on('error', () => {}).destroy()
+            result.body.on('error', nop).destroy()
           }
         },
         new CacheHandler(globalOpts, cacheKey, handler),
@@ -336,7 +357,7 @@ function handleResult (
 
   // Dump request body.
   if (util.isStream(opts.body)) {
-    opts.body.on('error', () => {}).destroy()
+    opts.body.on('error', nop).destroy()
   }
 
   sendCachedValue(handler, opts, result, age, null, false)
@@ -405,18 +426,17 @@ module.exports = (opts = {}) => {
       const result = store.get(cacheKey)
 
       if (result && typeof result.then === 'function') {
-        result.then(result => {
-          handleResult(dispatch,
+        return result
+          .then(result => handleResult(dispatch,
             globalOpts,
             cacheKey,
             handler,
             opts,
             reqCacheControl,
             result
-          )
-        })
+          ))
       } else {
-        handleResult(
+        return handleResult(
           dispatch,
           globalOpts,
           cacheKey,
@@ -426,8 +446,6 @@ module.exports = (opts = {}) => {
           result
         )
       }
-
-      return true
     }
   }
 }

package/lib/interceptor/decompress.js

@@ -3,6 +3,7 @@
 const { createInflate, createGunzip, createBrotliDecompress, createZstdDecompress } = require('node:zlib')
 const { pipeline } = require('node:stream')
 const DecoratorHandler = require('../handler/decorator-handler')
+const { runtimeFeatures } = require('../util/runtime-features')
 
 /** @typedef {import('node:stream').Transform} Transform */
 /** @typedef {import('node:stream').Transform} Controller */
@@ -16,7 +17,7 @@ const supportedEncodings = {
   deflate: createInflate,
   compress: createInflate,
   'x-compress': createInflate,
-  ...(createZstdDecompress ? { zstd: createZstdDecompress } : {})
+  ...(runtimeFeatures.has('zstd') ? { zstd: createZstdDecompress } : {})
 }
 
 const defaultSkipStatusCodes = /** @type {const} */ ([204, 304])

package/lib/interceptor/deduplicate.js

@@ -0,0 +1,109 @@
+'use strict'
+
+const diagnosticsChannel = require('node:diagnostics_channel')
+const util = require('../core/util')
+const DeduplicationHandler = require('../handler/deduplication-handler')
+const { normalizeHeaders, makeCacheKey, makeDeduplicationKey } = require('../util/cache.js')
+
+const pendingRequestsChannel = diagnosticsChannel.channel('undici:request:pending-requests')
+
+/**
+ * @param {import('../../types/interceptors.d.ts').default.DeduplicateInterceptorOpts} [opts]
+ * @returns {import('../../types/dispatcher.d.ts').default.DispatcherComposeInterceptor}
+ */
+module.exports = (opts = {}) => {
+  const {
+    methods = ['GET'],
+    skipHeaderNames = [],
+    excludeHeaderNames = []
+  } = opts
+
+  if (typeof opts !== 'object' || opts === null) {
+    throw new TypeError(`expected type of opts to be an Object, got ${opts === null ? 'null' : typeof opts}`)
+  }
+
+  if (!Array.isArray(methods)) {
+    throw new TypeError(`expected opts.methods to be an array, got ${typeof methods}`)
+  }
+
+  for (const method of methods) {
+    if (!util.safeHTTPMethods.includes(method)) {
+      throw new TypeError(`expected opts.methods to only contain safe HTTP methods, got ${method}`)
+    }
+  }
+
+  if (!Array.isArray(skipHeaderNames)) {
+    throw new TypeError(`expected opts.skipHeaderNames to be an array, got ${typeof skipHeaderNames}`)
+  }
+
+  if (!Array.isArray(excludeHeaderNames)) {
+    throw new TypeError(`expected opts.excludeHeaderNames to be an array, got ${typeof excludeHeaderNames}`)
+  }
+
+  // Convert to lowercase Set for case-insensitive header matching
+  const skipHeaderNamesSet = new Set(skipHeaderNames.map(name => name.toLowerCase()))
+
+  // Convert to lowercase Set for case-insensitive header exclusion from deduplication key
+  const excludeHeaderNamesSet = new Set(excludeHeaderNames.map(name => name.toLowerCase()))
+
+  const safeMethodsToNotDeduplicate = util.safeHTTPMethods.filter(method => methods.includes(method) === false)
+
+  /**
+   * Map of pending requests for deduplication
+   * @type {Map<string, DeduplicationHandler>}
+   */
+  const pendingRequests = new Map()
+
+  return dispatch => {
+    return (opts, handler) => {
+      if (!opts.origin || safeMethodsToNotDeduplicate.includes(opts.method)) {
+        return dispatch(opts, handler)
+      }
+
+      opts = {
+        ...opts,
+        headers: normalizeHeaders(opts)
+      }
+
+      // Skip deduplication if request contains any of the specified headers
+      if (skipHeaderNamesSet.size > 0) {
+        for (const headerName of Object.keys(opts.headers)) {
+          if (skipHeaderNamesSet.has(headerName.toLowerCase())) {
+            return dispatch(opts, handler)
+          }
+        }
+      }
+
+      const cacheKey = makeCacheKey(opts)
+      const dedupeKey = makeDeduplicationKey(cacheKey, excludeHeaderNamesSet)
+
+      // Check if there's already a pending request for this key
+      const pendingHandler = pendingRequests.get(dedupeKey)
+      if (pendingHandler) {
+        // Add this handler to the waiting list
+        pendingHandler.addWaitingHandler(handler)
+        return true
+      }
+
+      // Create a new deduplication handler
+      const deduplicationHandler = new DeduplicationHandler(
+        handler,
+        () => {
+          // Clean up when request completes
+          pendingRequests.delete(dedupeKey)
+          if (pendingRequestsChannel.hasSubscribers) {
+            pendingRequestsChannel.publish({ size: pendingRequests.size, key: dedupeKey, type: 'removed' })
+          }
+        }
+      )
+
+      // Register the pending request
+      pendingRequests.set(dedupeKey, deduplicationHandler)
+      if (pendingRequestsChannel.hasSubscribers) {
+        pendingRequestsChannel.publish({ size: pendingRequests.size, key: dedupeKey, type: 'added' })
+      }
+
+      return dispatch(opts, deduplicationHandler)
+    }
+  }
+}

package/lib/interceptor/dns.js

@@ -5,14 +5,44 @@ const DecoratorHandler = require('../handler/decorator-handler')
 const { InvalidArgumentError, InformationalError } = require('../core/errors')
 const maxInt = Math.pow(2, 31) - 1
 
+class DNSStorage {
+  #maxItems = 0
+  #records = new Map()
+
+  constructor (opts) {
+    this.#maxItems = opts.maxItems
+  }
+
+  get size () {
+    return this.#records.size
+  }
+
+  get (hostname) {
+    return this.#records.get(hostname) ?? null
+  }
+
+  set (hostname, records) {
+    this.#records.set(hostname, records)
+  }
+
+  delete (hostname) {
+    this.#records.delete(hostname)
+  }
+
+  // Delegate to storage decide can we do more lookups or not
+  full () {
+    return this.size >= this.#maxItems
+  }
+}
+
 class DNSInstance {
   #maxTTL = 0
   #maxItems = 0
-  #records = new Map()
   dualStack = true
   affinity = null
   lookup = null
   pick = null
+  storage = null
 
   constructor (opts) {
     this.#maxTTL = opts.maxTTL
@@ -21,17 +51,14 @@ class DNSInstance {
     this.affinity = opts.affinity
     this.lookup = opts.lookup ?? this.#defaultLookup
     this.pick = opts.pick ?? this.#defaultPick
-  }
-
-  get full () {
-    return this.#records.size === this.#maxItems
+    this.storage = opts.storage ?? new DNSStorage(opts)
   }
 
   runLookup (origin, opts, cb) {
-    const ips = this.#records.get(origin.hostname)
+    const ips = this.storage.get(origin.hostname)
 
     // If full, we just return the origin
-    if (ips == null && this.full) {
+    if (ips == null && this.storage.full()) {
       cb(null, origin)
       return
     }
@@ -55,7 +82,7 @@ class DNSInstance {
         }
 
         this.setRecords(origin, addresses)
-        const records = this.#records.get(origin.hostname)
+        const records = this.storage.get(origin.hostname)
 
         const ip = this.pick(
           origin,
@@ -89,7 +116,7 @@ class DNSInstance {
 
       // If no IPs we lookup - deleting old records
       if (ip == null) {
-        this.#records.delete(origin.hostname)
+        this.storage.delete(origin.hostname)
         this.runLookup(origin, opts, cb)
         return
       }
@@ -193,7 +220,7 @@ class DNSInstance {
   }
 
   pickFamily (origin, ipFamily) {
-    const records = this.#records.get(origin.hostname)?.records
+    const records = this.storage.get(origin.hostname)?.records
     if (!records) {
       return null
     }
@@ -227,11 +254,13 @@ class DNSInstance {
   setRecords (origin, addresses) {
     const timestamp = Date.now()
     const records = { records: { 4: null, 6: null } }
+    let minTTL = this.#maxTTL
     for (const record of addresses) {
       record.timestamp = timestamp
       if (typeof record.ttl === 'number') {
         // The record TTL is expected to be in ms
         record.ttl = Math.min(record.ttl, this.#maxTTL)
+        minTTL = Math.min(minTTL, record.ttl)
       } else {
         record.ttl = this.#maxTTL
       }
@@ -242,11 +271,12 @@ class DNSInstance {
       records.records[record.family] = familyRecords
     }
 
-    this.#records.set(origin.hostname, records)
+    // We provide a default TTL if external storage will be used without TTL per record-level support
+    this.storage.set(origin.hostname, records, { ttl: minTTL })
   }
 
   deleteRecords (origin) {
-    this.#records.delete(origin.hostname)
+    this.storage.delete(origin.hostname)
   }
 
   getHandler (meta, opts) {
@@ -372,6 +402,17 @@ module.exports = interceptorOpts => {
     throw new InvalidArgumentError('Invalid pick. Must be a function')
   }
 
+  if (
+    interceptorOpts?.storage != null &&
+    (typeof interceptorOpts?.storage?.get !== 'function' ||
+      typeof interceptorOpts?.storage?.set !== 'function' ||
+      typeof interceptorOpts?.storage?.full !== 'function' ||
+      typeof interceptorOpts?.storage?.delete !== 'function'
+    )
+  ) {
+    throw new InvalidArgumentError('Invalid storage. Must be a object with methods: { get, set, full, delete }')
+  }
+
   const dualStack = interceptorOpts?.dualStack ?? true
   let affinity
   if (dualStack) {
@@ -386,7 +427,8 @@ module.exports = interceptorOpts => {
     pick: interceptorOpts?.pick ?? null,
     dualStack,
     affinity,
-    maxItems: interceptorOpts?.maxItems ?? Infinity
+    maxItems: interceptorOpts?.maxItems ?? Infinity,
+    storage: interceptorOpts?.storage
   }
 
   const instance = new DNSInstance(opts)

package/lib/mock/mock-utils.js

@@ -337,8 +337,7 @@ function mockDispatch (opts, handler) {
       // synchronously throw the error, which breaks some tests.
       // Rather, we wait for the callback to resolve if it is a
       // promise, and then re-run handleReply with the new body.
-      body.then((newData) => handleReply(mockDispatches, newData))
-      return
+      return body.then((newData) => handleReply(mockDispatches, newData))
     }
 
     const responseData = getResponseData(body)

package/lib/mock/snapshot-agent.js

@@ -64,7 +64,9 @@ class SnapshotAgent extends MockAgent {
     this[kSnapshotLoaded] = false
 
     // For recording/update mode, we need a real agent to make actual requests
-    if (this[kSnapshotMode] === 'record' || this[kSnapshotMode] === 'update') {
+    // For playback mode, we need a real agent if there are excluded URLs
+    if (this[kSnapshotMode] === 'record' || this[kSnapshotMode] === 'update' ||
+        (this[kSnapshotMode] === 'playback' && opts.excludeUrls && opts.excludeUrls.length > 0)) {
       this[kRealAgent] = new Agent(opts)
     }
 
@@ -80,6 +82,12 @@ class SnapshotAgent extends MockAgent {
     handler = WrapHandler.wrap(handler)
     const mode = this[kSnapshotMode]
 
+    // Check if URL should be excluded (pass through without mocking/recording)
+    if (this[kSnapshotRecorder].isUrlExcluded(opts)) {
+      // Real agent is guaranteed by constructor when excludeUrls is configured
+      return this[kRealAgent].dispatch(opts, handler)
+    }
+
     if (mode === 'playback' || mode === 'update') {
       // Ensure snapshots are loaded
       if (!this[kSnapshotLoaded]) {
@@ -162,11 +170,9 @@ class SnapshotAgent extends MockAgent {
           headers: responseData.headers,
           body: responseBody,
           trailers: responseData.trailers
-        }).then(() => {
-          handler.onResponseEnd(controller, trailers)
-        }).catch((error) => {
-          handler.onResponseError(controller, error)
         })
+          .then(() => handler.onResponseEnd(controller, trailers))
+          .catch((error) => handler.onResponseError(controller, error))
       }
     }
 

package/lib/mock/snapshot-recorder.js

@@ -283,8 +283,7 @@ class SnapshotRecorder {
     }
 
     // Check URL exclusion patterns
-    const url = new URL(requestOpts.path, requestOpts.origin).toString()
-    if (this.#isUrlExcluded(url)) {
+    if (this.isUrlExcluded(requestOpts)) {
       return // Skip recording
     }
 
@@ -330,6 +329,16 @@ class SnapshotRecorder {
     }
   }
 
+  /**
+   * Checks if a URL should be excluded from recording/playback
+   * @param {SnapshotRequestOptions} requestOpts - Request options to check
+   * @returns {boolean} - True if URL is excluded
+   */
+  isUrlExcluded (requestOpts) {
+    const url = new URL(requestOpts.path, requestOpts.origin).toString()
+    return this.#isUrlExcluded(url)
+  }
+
   /**
    * Finds a matching snapshot for the given request
    * Returns the appropriate response based on call count for sequential responses
@@ -344,8 +353,7 @@ class SnapshotRecorder {
     }
 
     // Check URL exclusion patterns
-    const url = new URL(requestOpts.path, requestOpts.origin).toString()
-    if (this.#isUrlExcluded(url)) {
+    if (this.isUrlExcluded(requestOpts)) {
       return undefined // Skip playback
     }
 

package/lib/mock/snapshot-utils.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const { InvalidArgumentError } = require('../core/errors')
+const { runtimeFeatures } = require('../util/runtime-features.js')
 
 /**
  * @typedef {Object} HeaderFilters
@@ -25,10 +26,9 @@ function createHeaderFilters (matchOptions = {}) {
   }
 }
 
-let crypto
-try {
-  crypto = require('node:crypto')
-} catch { /* Fallback if crypto is not available */ }
+const crypto = runtimeFeatures.has('crypto')
+  ? require('node:crypto')
+  : null
 
 /**
  * @callback HashIdFunction

package/lib/util/cache.js

@@ -364,6 +364,33 @@ function assertCacheMethods (methods, name = 'CacheMethods') {
   }
 }
 
+/**
+ * Creates a string key for request deduplication purposes.
+ * This key is used to identify in-flight requests that can be shared.
+ * @param {import('../../types/cache-interceptor.d.ts').default.CacheKey} cacheKey
+ * @param {Set<string>} [excludeHeaders] Set of lowercase header names to exclude from the key
+ * @returns {string}
+ */
+function makeDeduplicationKey (cacheKey, excludeHeaders) {
+  // Create a deterministic string key from the cache key
+  // Include origin, method, path, and sorted headers
+  let key = `${cacheKey.origin}:${cacheKey.method}:${cacheKey.path}`
+
+  if (cacheKey.headers) {
+    const sortedHeaders = Object.keys(cacheKey.headers).sort()
+    for (const header of sortedHeaders) {
+      // Skip excluded headers
+      if (excludeHeaders?.has(header.toLowerCase())) {
+        continue
+      }
+      const value = cacheKey.headers[header]
+      key += `:${header}=${Array.isArray(value) ? value.join(',') : value}`
+    }
+  }
+
+  return key
+}
+
 module.exports = {
   makeCacheKey,
   normalizeHeaders,
@@ -373,5 +400,6 @@ module.exports = {
   parseVaryHeader,
   isEtagUsable,
   assertCacheMethods,
-  assertCacheStore
+  assertCacheStore,
+  makeDeduplicationKey
 }

package/lib/util/runtime-features.js

@@ -0,0 +1,124 @@
+'use strict'
+
+/** @typedef {`node:${string}`} NodeModuleName */
+
+/** @type {Record<NodeModuleName, () => any>} */
+const lazyLoaders = {
+  __proto__: null,
+  'node:crypto': () => require('node:crypto'),
+  'node:sqlite': () => require('node:sqlite'),
+  'node:worker_threads': () => require('node:worker_threads'),
+  'node:zlib': () => require('node:zlib')
+}
+
+/**
+ * @param {NodeModuleName} moduleName
+ * @returns {boolean}
+ */
+function detectRuntimeFeatureByNodeModule (moduleName) {
+  try {
+    lazyLoaders[moduleName]()
+    return true
+  } catch (err) {
+    if (err.code !== 'ERR_UNKNOWN_BUILTIN_MODULE') {
+      throw err
+    }
+    return false
+  }
+}
+
+/**
+ * @param {NodeModuleName} moduleName
+ * @param {string} property
+ * @returns {boolean}
+ */
+function detectRuntimeFeatureByExportedProperty (moduleName, property) {
+  const module = lazyLoaders[moduleName]()
+  return typeof module[property] !== 'undefined'
+}
+
+const runtimeFeaturesByExportedProperty = /** @type {const} */ (['markAsUncloneable', 'zstd'])
+
+/** @type {Record<RuntimeFeatureByExportedProperty, [NodeModuleName, string]>} */
+const exportedPropertyLookup = {
+  markAsUncloneable: ['node:worker_threads', 'markAsUncloneable'],
+  zstd: ['node:zlib', 'createZstdDecompress']
+}
+
+/** @typedef {typeof runtimeFeaturesByExportedProperty[number]} RuntimeFeatureByExportedProperty */
+
+const runtimeFeaturesAsNodeModule = /** @type {const} */ (['crypto', 'sqlite'])
+/** @typedef {typeof runtimeFeaturesAsNodeModule[number]} RuntimeFeatureByNodeModule */
+
+const features = /** @type {const} */ ([
+  ...runtimeFeaturesAsNodeModule,
+  ...runtimeFeaturesByExportedProperty
+])
+
+/** @typedef {typeof features[number]} Feature */
+
+/**
+ * @param {Feature} feature
+ * @returns {boolean}
+ */
+function detectRuntimeFeature (feature) {
+  if (runtimeFeaturesAsNodeModule.includes(/** @type {RuntimeFeatureByNodeModule} */ (feature))) {
+    return detectRuntimeFeatureByNodeModule(`node:${feature}`)
+  } else if (runtimeFeaturesByExportedProperty.includes(/** @type {RuntimeFeatureByExportedProperty} */ (feature))) {
+    const [moduleName, property] = exportedPropertyLookup[feature]
+    return detectRuntimeFeatureByExportedProperty(moduleName, property)
+  }
+  throw new TypeError(`unknown feature: ${feature}`)
+}
+
+/**
+ * @class
+ * @name RuntimeFeatures
+ */
+class RuntimeFeatures {
+  /** @type {Map<Feature, boolean>} */
+  #map = new Map()
+
+  /**
+   * Clears all cached feature detections.
+   */
+  clear () {
+    this.#map.clear()
+  }
+
+  /**
+   * @param {Feature} feature
+   * @returns {boolean}
+   */
+  has (feature) {
+    return (
+      this.#map.get(feature) ?? this.#detectRuntimeFeature(feature)
+    )
+  }
+
+  /**
+   * @param {Feature} feature
+   * @param {boolean} value
+   */
+  set (feature, value) {
+    if (features.includes(feature) === false) {
+      throw new TypeError(`unknown feature: ${feature}`)
+    }
+    this.#map.set(feature, value)
+  }
+
+  /**
+   * @param {Feature} feature
+   * @returns {boolean}
+   */
+  #detectRuntimeFeature (feature) {
+    const result = detectRuntimeFeature(feature)
+    this.#map.set(feature, result)
+    return result
+  }
+}
+
+const instance = new RuntimeFeatures()
+
+module.exports.runtimeFeatures = instance
+module.exports.default = instance

package/lib/web/cookies/parse.js

@@ -1,8 +1,8 @@
 'use strict'
 
+const { collectASequenceOfCodePointsFast } = require('../infra')
 const { maxNameValuePairSize, maxAttributeValueSize } = require('./constants')
 const { isCTLExcludingHtab } = require('./util')
-const { collectASequenceOfCodePointsFast } = require('../fetch/data-url')
 const assert = require('node:assert')
 const { unescape: qsUnescape } = require('node:querystring')