undici 7.0.0-alpha.3 → 7.0.0-alpha.5

package/lib/handler/redirect-handler.js

@@ -75,7 +75,8 @@ class RedirectHandler {
       this.opts.body &&
       typeof this.opts.body !== 'string' &&
       !ArrayBuffer.isView(this.opts.body) &&
-      util.isIterable(this.opts.body)
+      util.isIterable(this.opts.body) &&
+      !util.isFormDataLike(this.opts.body)
     ) {
       // TODO: Should we allow re-using iterable if !this.opts.idempotent
       // or through some other flag?
@@ -227,9 +228,10 @@ function cleanRequestHeaders (headers, removeContent, unknownOrigin) {
       }
     }
   } else if (headers && typeof headers === 'object') {
-    for (const key of Object.keys(headers)) {
+    const entries = typeof headers[Symbol.iterator] === 'function' ? headers : Object.entries(headers)
+    for (const [key, value] of entries) {
       if (!shouldRemoveHeader(key, removeContent, unknownOrigin)) {
-        ret.push(key, headers[key])
+        ret.push(key, value)
       }
     }
   } else {

package/lib/handler/retry-handler.js

@@ -229,7 +229,7 @@ class RetryHandler {
         return false
       }
 
-      const { start, size, end = size } = contentRange
+      const { start, size, end = size - 1 } = contentRange
 
       assert(this.start === start, 'content-range mismatch')
       assert(this.end == null || this.end === end, 'content-range mismatch')
@@ -252,7 +252,7 @@ class RetryHandler {
           )
         }
 
-        const { start, size, end = size } = range
+        const { start, size, end = size - 1 } = range
         assert(
           start != null && Number.isFinite(start),
           'content-range mismatch'
@@ -266,7 +266,7 @@ class RetryHandler {
       // We make our best to checkpoint the body for further range headers
       if (this.end == null) {
         const contentLength = headers['content-length']
-        this.end = contentLength != null ? Number(contentLength) : null
+        this.end = contentLength != null ? Number(contentLength) - 1 : null
       }
 
       assert(Number.isFinite(this.start))

package/lib/interceptor/cache.js

@@ -1,13 +1,88 @@
 'use strict'
 
+const assert = require('node:assert')
+const { Readable } = require('node:stream')
 const util = require('../core/util')
 const CacheHandler = require('../handler/cache-handler')
 const MemoryCacheStore = require('../cache/memory-cache-store')
 const CacheRevalidationHandler = require('../handler/cache-revalidation-handler')
-const { assertCacheStore, assertCacheMethods } = require('../util/cache.js')
+const { assertCacheStore, assertCacheMethods, makeCacheKey, parseCacheControlHeader } = require('../util/cache.js')
+const { nowAbsolute } = require('../util/timers.js')
 
 const AGE_HEADER = Buffer.from('age')
 
+/**
+ * @param {import('../../types/dispatcher.d.ts').default.DispatchHandlers} handler
+ */
+function sendGatewayTimeout (handler) {
+  let aborted = false
+  try {
+    if (typeof handler.onConnect === 'function') {
+      handler.onConnect(() => {
+        aborted = true
+      })
+
+      if (aborted) {
+        return
+      }
+    }
+
+    if (typeof handler.onHeaders === 'function') {
+      handler.onHeaders(504, [], () => {}, 'Gateway Timeout')
+      if (aborted) {
+        return
+      }
+    }
+
+    if (typeof handler.onComplete === 'function') {
+      handler.onComplete([])
+    }
+  } catch (err) {
+    if (typeof handler.onError === 'function') {
+      handler.onError(err)
+    }
+  }
+}
+
+/**
+ * @param {import('../../types/cache-interceptor.d.ts').default.GetResult} result
+ * @param {number} age
+ * @param {import('../util/cache.js').CacheControlDirectives | undefined} cacheControlDirectives
+ * @returns {boolean}
+ */
+function needsRevalidation (result, age, cacheControlDirectives) {
+  if (cacheControlDirectives?.['no-cache']) {
+    // Always revalidate requests with the no-cache directive
+    return true
+  }
+
+  const now = nowAbsolute()
+  if (now > result.staleAt) {
+    // Response is stale
+    if (cacheControlDirectives?.['max-stale']) {
+      // There's a threshold where we can serve stale responses, let's see if
+      //  we're in it
+      // https://www.rfc-editor.org/rfc/rfc9111.html#name-max-stale
+      const gracePeriod = result.staleAt + (cacheControlDirectives['max-stale'] * 1000)
+      return now > gracePeriod
+    }
+
+    return true
+  }
+
+  if (cacheControlDirectives?.['min-fresh']) {
+    // https://www.rfc-editor.org/rfc/rfc9111.html#section-5.2.1.3
+
+    // At this point, staleAt is always > now
+    const timeLeftTillStale = result.staleAt - now
+    const threshold = cacheControlDirectives['min-fresh'] * 1000
+
+    return timeLeftTillStale <= threshold
+  }
+
+  return false
+}
+
 /**
  * @param {import('../../types/cache-interceptor.d.ts').default.CacheOptions} [opts]
  * @returns {import('../../types/dispatcher.d.ts').default.DispatcherComposeInterceptor}
@@ -34,136 +109,182 @@ module.exports = (opts = {}) => {
 
   return dispatch => {
     return (opts, handler) => {
+      // TODO (fix): What if e.g. opts.headers has if-modified-since header? Or other headers
+      // that make things ambigious?
+
       if (!opts.origin || safeMethodsToNotCache.includes(opts.method)) {
         // Not a method we want to cache or we don't have the origin, skip
         return dispatch(opts, handler)
       }
 
-      const stream = store.createReadStream(opts)
-      if (!stream) {
-        // Request isn't cached
-        return dispatch(opts, new CacheHandler(globalOpts, opts, handler))
-      }
+      const requestCacheControl = opts.headers?.['cache-control']
+        ? parseCacheControlHeader(opts.headers['cache-control'])
+        : undefined
 
-      let onErrorCalled = false
+      if (requestCacheControl?.['no-store']) {
+        return dispatch(opts, handler)
+      }
 
       /**
-       * @param {import('../../types/cache-interceptor.d.ts').default.CacheStoreReadable} stream
-       * @param {import('../../types/cache-interceptor.d.ts').default.CacheStoreValue} value
+       * @type {import('../../types/cache-interceptor.d.ts').default.CacheKey}
        */
-      const respondWithCachedValue = (stream, value) => {
-        const ac = new AbortController()
-        const signal = ac.signal
+      const cacheKey = makeCacheKey(opts)
 
-        signal.onabort = (_, err) => {
-          stream.destroy()
-          if (!onErrorCalled) {
-            handler.onError(err)
-            onErrorCalled = true
-          }
+      // TODO (perf): For small entries support returning a Buffer instead of a stream.
+      // Maybe store should return { staleAt, headers, body, etc... } instead of a stream + stream.value?
+      // Where body can be a Buffer, string, stream or blob?
+      const result = store.get(cacheKey)
+      if (!result) {
+        if (requestCacheControl?.['only-if-cached']) {
+          // We only want cached responses
+          //  https://www.rfc-editor.org/rfc/rfc9111.html#name-only-if-cached
+          sendGatewayTimeout(handler)
+          return true
         }
 
-        stream.on('error', (err) => {
-          if (!onErrorCalled) {
-            handler.onError(err)
-            onErrorCalled = true
-          }
-        })
-
-        try {
-          if (typeof handler.onConnect === 'function') {
-            handler.onConnect(ac.abort)
-            signal.throwIfAborted()
-          }
-
-          if (typeof handler.onHeaders === 'function') {
-            // Add the age header
-            // https://www.rfc-editor.org/rfc/rfc9111.html#name-age
-            const age = Math.round((Date.now() - value.cachedAt) / 1000)
+        return dispatch(opts, new CacheHandler(globalOpts, cacheKey, handler))
+      }
 
-            value.rawHeaders.push(AGE_HEADER, Buffer.from(`${age}`))
+      /**
+       * @param {import('../../types/cache-interceptor.d.ts').default.GetResult} result
+       * @param {number} age
+       */
+      const respondWithCachedValue = ({ rawHeaders, statusCode, statusMessage, body }, age) => {
+        const stream = util.isStream(body)
+          ? body
+          : Readable.from(body ?? [])
 
-            handler.onHeaders(value.statusCode, value.rawHeaders, stream.resume, value.statusMessage)
-            signal.throwIfAborted()
-          }
+        assert(!stream.destroyed, 'stream should not be destroyed')
+        assert(!stream.readableDidRead, 'stream should not be readableDidRead')
 
-          if (opts.method === 'HEAD') {
-            if (typeof handler.onComplete === 'function') {
-              handler.onComplete(null)
-              stream.destroy()
+        stream
+          .on('error', function (err) {
+            if (!this.readableEnded) {
+              if (typeof handler.onError === 'function') {
+                handler.onError(err)
+              } else {
+                throw err
+              }
             }
-          } else {
-            if (typeof handler.onData === 'function') {
-              stream.on('data', chunk => {
-                if (!handler.onData(chunk)) {
-                  stream.pause()
-                }
-              })
+          })
+          .on('close', function () {
+            if (!this.errored && typeof handler.onComplete === 'function') {
+              handler.onComplete([])
             }
+          })
 
-            if (typeof handler.onComplete === 'function') {
-              stream.on('end', () => {
-                handler.onComplete(value.rawTrailers ?? [])
-              })
-            }
+        if (typeof handler.onConnect === 'function') {
+          handler.onConnect((err) => {
+            stream.destroy(err)
+          })
+
+          if (stream.destroyed) {
+            return
           }
-        } catch (err) {
-          stream.destroy(err)
-          if (!onErrorCalled && typeof handler.onError === 'function') {
-            handler.onError(err)
-            onErrorCalled = true
+        }
+
+        if (typeof handler.onHeaders === 'function') {
+          // Add the age header
+          // https://www.rfc-editor.org/rfc/rfc9111.html#name-age
+
+          // TODO (fix): What if rawHeaders already contains age header?
+          rawHeaders = [...rawHeaders, AGE_HEADER, Buffer.from(`${age}`)]
+
+          if (handler.onHeaders(statusCode, rawHeaders, () => stream?.resume(), statusMessage) === false) {
+            stream.pause()
           }
         }
+
+        if (opts.method === 'HEAD') {
+          stream.destroy()
+        } else {
+          stream.on('data', function (chunk) {
+            if (typeof handler.onData === 'function' && !handler.onData(chunk)) {
+              stream.pause()
+            }
+          })
+        }
       }
 
       /**
-       * @param {import('../../types/cache-interceptor.d.ts').default.CacheStoreReadable | undefined} stream
+       * @param {import('../../types/cache-interceptor.d.ts').default.GetResult} result
        */
-      const handleStream = (stream) => {
-        if (!stream) {
-          // Request isn't cached
-          return dispatch(opts, new CacheHandler(globalOpts, opts, handler))
-        }
+      const handleResult = (result) => {
+        // TODO (perf): Readable.from path can be optimized...
 
-        const { value } = stream
+        if (!result.body && opts.method !== 'HEAD') {
+          throw new Error('stream is undefined but method isn\'t HEAD')
+        }
 
-        // Dump body on error
-        if (util.isStream(opts.body)) {
-          opts.body?.on('error', () => {}).resume()
+        const age = Math.round((nowAbsolute() - result.cachedAt) / 1000)
+        if (requestCacheControl?.['max-age'] && age >= requestCacheControl['max-age']) {
+          // Response is considered expired for this specific request
+          //  https://www.rfc-editor.org/rfc/rfc9111.html#section-5.2.1.1
+          return dispatch(opts, handler)
         }
 
         // Check if the response is stale
-        const now = Date.now()
-        if (now >= value.staleAt) {
-          if (now >= value.deleteAt) {
-            // Safety check in case the store gave us a response that should've been
-            //  deleted already
-            dispatch(opts, new CacheHandler(globalOpts, opts, handler))
-            return
-          }
-
-          if (!opts.headers) {
-            opts.headers = {}
+        if (needsRevalidation(result, age, requestCacheControl)) {
+          if (util.isStream(opts.body) && util.bodyLength(opts.body) !== 0) {
+            // If body is is stream we can't revalidate...
+            // TODO (fix): This could be less strict...
+            return dispatch(opts, new CacheHandler(globalOpts, cacheKey, handler))
           }
 
-          opts.headers['if-modified-since'] = new Date(value.cachedAt).toUTCString()
-
-          // Need to revalidate the response
-          dispatch(
-            opts,
+          // We need to revalidate the response
+          return dispatch(
+            {
+              ...opts,
+              headers: {
+                ...opts.headers,
+                'if-modified-since': new Date(result.cachedAt).toUTCString(),
+                etag: result.etag
+              }
+            },
             new CacheRevalidationHandler(
-              () => respondWithCachedValue(stream, value),
-              new CacheHandler(globalOpts, opts, handler)
+              (success) => {
+                if (success) {
+                  respondWithCachedValue(result, age)
+                } else if (util.isStream(result.body)) {
+                  result.body.on('error', () => {}).destroy()
+                }
+              },
+              new CacheHandler(globalOpts, cacheKey, handler)
             )
           )
-
-          return
         }
 
-        respondWithCachedValue(stream, value)
+        // Dump request body.
+        if (util.isStream(opts.body)) {
+          opts.body.on('error', () => {}).destroy()
+        }
+        respondWithCachedValue(result, age)
       }
 
-      Promise.resolve(stream).then(handleStream).catch(handler.onError)
+      if (typeof result.then === 'function') {
+        result.then((result) => {
+          if (!result) {
+            if (requestCacheControl?.['only-if-cached']) {
+              // We only want cached responses
+              //  https://www.rfc-editor.org/rfc/rfc9111.html#name-only-if-cached
+              sendGatewayTimeout(handler)
+              return true
+            }
+
+            dispatch(opts, new CacheHandler(globalOpts, cacheKey, handler))
+          } else {
+            handleResult(result)
+          }
+        }, err => {
+          if (typeof handler.onError === 'function') {
+            handler.onError(err)
+          } else {
+            throw err
+          }
+        })
+      } else {
+        handleResult(result)
+      }
 
       return true
     }

package/lib/interceptor/dns.js

@@ -13,7 +13,6 @@ class DNSInstance {
   affinity = null
   lookup = null
   pick = null
-  lastIpFamily = null
 
   constructor (opts) {
     this.#maxTTL = opts.maxTTL
@@ -61,16 +60,23 @@ class DNSInstance {
         const ip = this.pick(
           origin,
           records,
-          // Only set affinity if dual stack is disabled
-          // otherwise let it go through normal flow
-          !newOpts.dualStack && newOpts.affinity
+          newOpts.affinity
         )
 
+        let port
+        if (typeof ip.port === 'number') {
+          port = `:${ip.port}`
+        } else if (origin.port !== '') {
+          port = `:${origin.port}`
+        } else {
+          port = ''
+        }
+
         cb(
           null,
           `${origin.protocol}//${
             ip.family === 6 ? `[${ip.address}]` : ip.address
-          }${origin.port === '' ? '' : `:${origin.port}`}`
+          }${port}`
         )
       })
     } else {
@@ -78,9 +84,7 @@ class DNSInstance {
       const ip = this.pick(
         origin,
         ips,
-        // Only set affinity if dual stack is disabled
-        // otherwise let it go through normal flow
-        !newOpts.dualStack && newOpts.affinity
+        newOpts.affinity
       )
 
       // If no IPs we lookup - deleting old records
@@ -90,11 +94,20 @@ class DNSInstance {
         return
       }
 
+      let port
+      if (typeof ip.port === 'number') {
+        port = `:${ip.port}`
+      } else if (origin.port !== '') {
+        port = `:${origin.port}`
+      } else {
+        port = ''
+      }
+
       cb(
         null,
         `${origin.protocol}//${
           ip.family === 6 ? `[${ip.address}]` : ip.address
-        }${origin.port === '' ? '' : `:${origin.port}`}`
+        }${port}`
       )
     }
   }
@@ -102,7 +115,11 @@ class DNSInstance {
   #defaultLookup (origin, opts, cb) {
     lookup(
       origin.hostname,
-      { all: true, family: this.dualStack === false ? this.affinity : 0 },
+      {
+        all: true,
+        family: this.dualStack === false ? this.affinity : 0,
+        order: 'ipv4first'
+      },
       (err, addresses) => {
         if (err) {
           return cb(err)
@@ -111,15 +128,9 @@ class DNSInstance {
         const results = new Map()
 
         for (const addr of addresses) {
-          const record = {
-            address: addr.address,
-            ttl: opts.maxTTL,
-            family: addr.family
-          }
-
           // On linux we found duplicates, we attempt to remove them with
           // the latest record
-          results.set(`${record.address}:${record.family}`, record)
+          results.set(`${addr.address}:${addr.family}`, addr)
         }
 
         cb(null, results.values())
@@ -129,36 +140,36 @@ class DNSInstance {
 
   #defaultPick (origin, hostnameRecords, affinity) {
     let ip = null
-    const { records, offset = 0 } = hostnameRecords
-    let newOffset = 0
+    const { records, offset } = hostnameRecords
+
+    let family
+    if (this.dualStack) {
+      if (affinity == null) {
+        // Balance between ip families
+        if (offset == null || offset === maxInt) {
+          hostnameRecords.offset = 0
+          affinity = 4
+        } else {
+          hostnameRecords.offset++
+          affinity = (hostnameRecords.offset & 1) === 1 ? 6 : 4
+        }
+      }
 
-    if (offset === maxInt) {
-      newOffset = 0
+      if (records[affinity] != null && records[affinity].ips.length > 0) {
+        family = records[affinity]
+      } else {
+        family = records[affinity === 4 ? 6 : 4]
+      }
     } else {
-      newOffset = offset + 1
+      family = records[affinity]
     }
 
-    // We balance between the two IP families
-    // If dual-stack disabled, we automatically pick the affinity
-    const newIpFamily = (newOffset & 1) === 1 ? 4 : 6
-    const family =
-      this.dualStack === false
-        ? records[this.affinity] // If dual-stack is disabled, we pick the default affiniy
-        : records[affinity] ?? records[newIpFamily]
-
-    // If no IPs and we have tried both families or dual stack is disabled, we return null
-    if (
-      (family == null || family.ips.length === 0) &&
-      // eslint-disable-next-line eqeqeq
-      (this.dualStack === false || this.lastIpFamily != newIpFamily)
-    ) {
+    // If no IPs we return null
+    if (family == null || family.ips.length === 0) {
       return ip
     }
 
-    family.offset = family.offset ?? 0
-    hostnameRecords.offset = newOffset
-
-    if (family.offset === maxInt) {
+    if (family.offset == null || family.offset === maxInt) {
       family.offset = 0
     } else {
       family.offset++
@@ -171,24 +182,28 @@ class DNSInstance {
       return ip
     }
 
-    const timestamp = Date.now()
-    // Record TTL is already in ms
-    if (ip.timestamp != null && timestamp - ip.timestamp > ip.ttl) {
+    if (Date.now() - ip.timestamp > ip.ttl) { // record TTL is already in ms
       // We delete expired records
       // It is possible that they have different TTL, so we manage them individually
       family.ips.splice(position, 1)
       return this.pick(origin, hostnameRecords, affinity)
     }
 
-    ip.timestamp = timestamp
-
-    this.lastIpFamily = newIpFamily
     return ip
   }
 
   setRecords (origin, addresses) {
+    const timestamp = Date.now()
     const records = { records: { 4: null, 6: null } }
     for (const record of addresses) {
+      record.timestamp = timestamp
+      if (typeof record.ttl === 'number') {
+        // The record TTL is expected to be in ms
+        record.ttl = Math.min(record.ttl, this.#maxTTL)
+      } else {
+        record.ttl = this.#maxTTL
+      }
+
       const familyRecords = records.records[record.family] ?? { ips: [] }
 
       familyRecords.ips.push(record)
@@ -302,12 +317,20 @@ module.exports = interceptorOpts => {
     throw new InvalidArgumentError('Invalid pick. Must be a function')
   }
 
+  const dualStack = interceptorOpts?.dualStack ?? true
+  let affinity
+  if (dualStack) {
+    affinity = interceptorOpts?.affinity ?? null
+  } else {
+    affinity = interceptorOpts?.affinity ?? 4
+  }
+
   const opts = {
     maxTTL: interceptorOpts?.maxTTL ?? 10e3, // Expressed in ms
     lookup: interceptorOpts?.lookup ?? null,
     pick: interceptorOpts?.pick ?? null,
-    dualStack: interceptorOpts?.dualStack ?? true,
-    affinity: interceptorOpts?.affinity ?? 4,
+    dualStack,
+    affinity,
     maxItems: interceptorOpts?.maxItems ?? Infinity
   }