undici 7.0.0-alpha.2 → 7.0.0-alpha.4

package/lib/handler/cache-handler.js

@@ -0,0 +1,389 @@
+'use strict'
+
+const util = require('../core/util')
+const DecoratorHandler = require('../handler/decorator-handler')
+const {
+  parseCacheControlHeader,
+  parseVaryHeader
+} = require('../util/cache')
+
+function noop () {}
+
+/**
+ * Writes a response to a CacheStore and then passes it on to the next handler
+ */
+class CacheHandler extends DecoratorHandler {
+  /**
+   * @type {import('../../types/cache-interceptor.d.ts').default.CacheKey}
+   */
+  #cacheKey
+
+  /**
+   * @type {import('../../types/cache-interceptor.d.ts').default.CacheStore}
+   */
+  #store
+
+  /**
+   * @type {import('../../types/dispatcher.d.ts').default.DispatchHandlers}
+   */
+  #handler
+
+  /**
+   * @type {import('node:stream').Writable | undefined}
+   */
+  #writeStream
+
+  /**
+   * @param {import('../../types/cache-interceptor.d.ts').default.CacheOptions} opts
+   * @param {import('../../types/cache-interceptor.d.ts').default.CacheKey} cacheKey
+   * @param {import('../../types/dispatcher.d.ts').default.DispatchHandlers} handler
+   */
+  constructor (opts, cacheKey, handler) {
+    const { store } = opts
+
+    super(handler)
+
+    this.#store = store
+    this.#cacheKey = cacheKey
+    this.#handler = handler
+  }
+
+  onConnect (abort) {
+    if (this.#writeStream) {
+      this.#writeStream.destroy()
+      this.#writeStream = undefined
+    }
+
+    if (typeof this.#handler.onConnect === 'function') {
+      this.#handler.onConnect(abort)
+    }
+  }
+
+  /**
+   * @see {DispatchHandlers.onHeaders}
+   *
+   * @param {number} statusCode
+   * @param {Buffer[]} rawHeaders
+   * @param {() => void} resume
+   * @param {string} statusMessage
+   * @returns {boolean}
+   */
+  onHeaders (
+    statusCode,
+    rawHeaders,
+    resume,
+    statusMessage
+  ) {
+    const downstreamOnHeaders = () => {
+      if (typeof this.#handler.onHeaders === 'function') {
+        return this.#handler.onHeaders(
+          statusCode,
+          rawHeaders,
+          resume,
+          statusMessage
+        )
+      } else {
+        return true
+      }
+    }
+
+    if (
+      !util.safeHTTPMethods.includes(this.#cacheKey.method) &&
+      statusCode >= 200 &&
+      statusCode <= 399
+    ) {
+      // https://www.rfc-editor.org/rfc/rfc9111.html#name-invalidating-stored-response
+      try {
+        this.#store.delete(this.#cacheKey).catch?.(noop)
+      } catch {
+        // Fail silently
+      }
+      return downstreamOnHeaders()
+    }
+
+    const parsedRawHeaders = util.parseRawHeaders(rawHeaders)
+    const headers = util.parseHeaders(parsedRawHeaders)
+
+    const cacheControlHeader = headers['cache-control']
+    const isCacheFull = typeof this.#store.isFull !== 'undefined'
+      ? this.#store.isFull
+      : false
+
+    if (
+      !cacheControlHeader ||
+      isCacheFull
+    ) {
+      // Don't have the cache control header or the cache is full
+      return downstreamOnHeaders()
+    }
+    const cacheControlDirectives = parseCacheControlHeader(cacheControlHeader)
+    if (!canCacheResponse(statusCode, headers, cacheControlDirectives)) {
+      return downstreamOnHeaders()
+    }
+
+    const now = Date.now()
+    const staleAt = determineStaleAt(now, headers, cacheControlDirectives)
+    if (staleAt) {
+      const varyDirectives = this.#cacheKey.headers && headers.vary
+        ? parseVaryHeader(headers.vary, this.#cacheKey.headers)
+        : undefined
+      const deleteAt = determineDeleteAt(now, cacheControlDirectives, staleAt)
+
+      const strippedHeaders = stripNecessaryHeaders(
+        rawHeaders,
+        parsedRawHeaders,
+        cacheControlDirectives
+      )
+
+      this.#writeStream = this.#store.createWriteStream(this.#cacheKey, {
+        statusCode,
+        statusMessage,
+        rawHeaders: strippedHeaders,
+        vary: varyDirectives,
+        cachedAt: now,
+        staleAt,
+        deleteAt
+      })
+
+      if (this.#writeStream) {
+        const handler = this
+        this.#writeStream
+          .on('drain', resume)
+          .on('error', function () {
+            // TODO (fix): Make error somehow observable?
+          })
+          .on('close', function () {
+            if (handler.#writeStream === this) {
+              handler.#writeStream = undefined
+            }
+
+            // TODO (fix): Should we resume even if was paused downstream?
+            resume()
+          })
+      }
+    }
+
+    return downstreamOnHeaders()
+  }
+
+  /**
+   * @see {DispatchHandlers.onData}
+   *
+   * @param {Buffer} chunk
+   * @returns {boolean}
+   */
+  onData (chunk) {
+    let paused = false
+
+    if (this.#writeStream) {
+      paused ||= this.#writeStream.write(chunk) === false
+    }
+
+    if (typeof this.#handler.onData === 'function') {
+      paused ||= this.#handler.onData(chunk) === false
+    }
+
+    return !paused
+  }
+
+  /**
+   * @see {DispatchHandlers.onComplete}
+   *
+   * @param {string[] | null} rawTrailers
+   */
+  onComplete (rawTrailers) {
+    if (this.#writeStream) {
+      this.#writeStream.end()
+    }
+
+    if (typeof this.#handler.onComplete === 'function') {
+      return this.#handler.onComplete(rawTrailers)
+    }
+  }
+
+  /**
+   * @see {DispatchHandlers.onError}
+   *
+   * @param {Error} err
+   */
+  onError (err) {
+    if (this.#writeStream) {
+      this.#writeStream.destroy(err)
+      this.#writeStream = undefined
+    }
+
+    if (typeof this.#handler.onError === 'function') {
+      this.#handler.onError(err)
+    }
+  }
+}
+
+/**
+ * @see https://www.rfc-editor.org/rfc/rfc9111.html#name-storing-responses-to-authen
+ *
+ * @param {number} statusCode
+ * @param {Record<string, string | string[]>} headers
+ * @param {import('../util/cache.js').CacheControlDirectives} cacheControlDirectives
+ */
+function canCacheResponse (statusCode, headers, cacheControlDirectives) {
+  if (
+    statusCode !== 200 &&
+    statusCode !== 307
+  ) {
+    return false
+  }
+
+  if (
+    cacheControlDirectives.private === true ||
+    cacheControlDirectives['no-cache'] === true ||
+    cacheControlDirectives['no-store']
+  ) {
+    return false
+  }
+
+  // https://www.rfc-editor.org/rfc/rfc9111.html#section-4.1-5
+  if (headers.vary === '*') {
+    return false
+  }
+
+  // https://www.rfc-editor.org/rfc/rfc9111.html#name-storing-responses-to-authen
+  if (headers.authorization) {
+    if (!cacheControlDirectives.public || typeof headers.authorization !== 'string') {
+      return false
+    }
+
+    if (
+      Array.isArray(cacheControlDirectives['no-cache']) &&
+      cacheControlDirectives['no-cache'].includes('authorization')
+    ) {
+      return false
+    }
+
+    if (
+      Array.isArray(cacheControlDirectives['private']) &&
+      cacheControlDirectives['private'].includes('authorization')
+    ) {
+      return false
+    }
+  }
+
+  return true
+}
+
+/**
+ * @param {number} now
+ * @param {Record<string, string | string[]>} headers
+ * @param {import('../util/cache.js').CacheControlDirectives} cacheControlDirectives
+ *
+ * @returns {number | undefined} time that the value is stale at or undefined if it shouldn't be cached
+ */
+function determineStaleAt (now, headers, cacheControlDirectives) {
+  // Prioritize s-maxage since we're a shared cache
+  //  s-maxage > max-age > Expire
+  //  https://www.rfc-editor.org/rfc/rfc9111.html#section-5.2.2.10-3
+  const sMaxAge = cacheControlDirectives['s-maxage']
+  if (sMaxAge) {
+    return now + (sMaxAge * 1000)
+  }
+
+  if (cacheControlDirectives.immutable) {
+    // https://www.rfc-editor.org/rfc/rfc8246.html#section-2.2
+    return now + 31536000
+  }
+
+  const maxAge = cacheControlDirectives['max-age']
+  if (maxAge) {
+    return now + (maxAge * 1000)
+  }
+
+  if (headers.expire && typeof headers.expire === 'string') {
+    // https://www.rfc-editor.org/rfc/rfc9111.html#section-5.3
+    const expiresDate = new Date(headers.expire)
+    if (expiresDate instanceof Date && !isNaN(expiresDate)) {
+      return now + (Date.now() - expiresDate.getTime())
+    }
+  }
+
+  return undefined
+}
+
+/**
+ * @param {number} now
+ * @param {import('../util/cache.js').CacheControlDirectives} cacheControlDirectives
+ * @param {number} staleAt
+ */
+function determineDeleteAt (now, cacheControlDirectives, staleAt) {
+  if (cacheControlDirectives['stale-while-revalidate']) {
+    return now + (cacheControlDirectives['stale-while-revalidate'] * 1000)
+  }
+
+  return staleAt
+}
+
+/**
+ * Strips headers required to be removed in cached responses
+ * @param {Buffer[]} rawHeaders
+ * @param {string[]} parsedRawHeaders
+ * @param {import('../util/cache.js').CacheControlDirectives} cacheControlDirectives
+ * @returns {Buffer[]}
+ */
+function stripNecessaryHeaders (rawHeaders, parsedRawHeaders, cacheControlDirectives) {
+  const headersToRemove = ['connection']
+
+  if (Array.isArray(cacheControlDirectives['no-cache'])) {
+    headersToRemove.push(...cacheControlDirectives['no-cache'])
+  }
+
+  if (Array.isArray(cacheControlDirectives['private'])) {
+    headersToRemove.push(...cacheControlDirectives['private'])
+  }
+
+  let strippedHeaders
+
+  let offset = 0
+  for (let i = 0; i < parsedRawHeaders.length; i += 2) {
+    const headerName = parsedRawHeaders[i]
+
+    if (headersToRemove.includes(headerName)) {
+      // We have at least one header we want to remove
+      if (!strippedHeaders) {
+        // This is the first header we want to remove, let's create the array
+        // Since we're stripping headers, this will over allocate. We'll trim
+        //  it later.
+        strippedHeaders = new Array(parsedRawHeaders.length)
+
+        // Backfill the previous headers into it
+        for (let j = 0; j < i; j += 2) {
+          strippedHeaders[j] = parsedRawHeaders[j]
+          strippedHeaders[j + 1] = parsedRawHeaders[j + 1]
+        }
+      }
+
+      // We can't map indices 1:1 from stripped headers to rawHeaders without
+      //  creating holes (if we skip a header, we now have two holes where at
+      //  element should be). So, let's keep an offset to keep strippedHeaders
+      //  flattened. We can also use this at the end for trimming the empty
+      //  elements off of strippedHeaders.
+      offset += 2
+
+      continue
+    }
+
+    // We want to keep this header. Let's add it to strippedHeaders if it exists
+    if (strippedHeaders) {
+      strippedHeaders[i - offset] = parsedRawHeaders[i]
+      strippedHeaders[i + 1 - offset] = parsedRawHeaders[i + 1]
+    }
+  }
+
+  if (strippedHeaders) {
+    // Trim off the empty values at the end
+    strippedHeaders.length -= offset
+  }
+
+  return strippedHeaders
+    ? util.encodeRawHeaders(strippedHeaders)
+    : rawHeaders
+}
+
+module.exports = CacheHandler

package/lib/handler/cache-revalidation-handler.js

@@ -0,0 +1,151 @@
+'use strict'
+
+const assert = require('node:assert')
+const DecoratorHandler = require('../handler/decorator-handler')
+
+/**
+ * This takes care of revalidation requests we send to the origin. If we get
+ *  a response indicating that what we have is cached (via a HTTP 304), we can
+ *  continue using the cached value. Otherwise, we'll receive the new response
+ *  here, which we then just pass on to the next handler (most likely a
+ *  CacheHandler). Note that this assumes the proper headers were already
+ *  included in the request to tell the origin that we want to revalidate the
+ *  response (i.e. if-modified-since).
+ *
+ * @see https://www.rfc-editor.org/rfc/rfc9111.html#name-validation
+ *
+ * @typedef {import('../../types/dispatcher.d.ts').default.DispatchHandlers} DispatchHandlers
+ * @implements {DispatchHandlers}
+ */
+class CacheRevalidationHandler extends DecoratorHandler {
+  #successful = false
+  /**
+   * @type {((boolean) => void) | null}
+   */
+  #callback
+  /**
+   * @type {(import('../../types/dispatcher.d.ts').default.DispatchHandlers)}
+   */
+  #handler
+
+  #abort
+
+  /**
+   * @param {(boolean) => void} callback Function to call if the cached value is valid
+   * @param {import('../../types/dispatcher.d.ts').default.DispatchHandlers} handler
+   */
+  constructor (callback, handler) {
+    if (typeof callback !== 'function') {
+      throw new TypeError('callback must be a function')
+    }
+
+    super(handler)
+
+    this.#callback = callback
+    this.#handler = handler
+  }
+
+  onConnect (abort) {
+    this.#successful = false
+    this.#abort = abort
+  }
+
+  /**
+   * @see {DispatchHandlers.onHeaders}
+   *
+   * @param {number} statusCode
+   * @param {Buffer[]} rawHeaders
+   * @param {() => void} resume
+   * @param {string} statusMessage
+   * @returns {boolean}
+   */
+  onHeaders (
+    statusCode,
+    rawHeaders,
+    resume,
+    statusMessage
+  ) {
+    assert(this.#callback != null)
+
+    // https://www.rfc-editor.org/rfc/rfc9111.html#name-handling-a-validation-respo
+    this.#successful = statusCode === 304
+    this.#callback(this.#successful)
+    this.#callback = null
+
+    if (this.#successful) {
+      return true
+    }
+
+    if (typeof this.#handler.onConnect === 'function') {
+      this.#handler.onConnect(this.#abort)
+    }
+
+    if (typeof this.#handler.onHeaders === 'function') {
+      return this.#handler.onHeaders(
+        statusCode,
+        rawHeaders,
+        resume,
+        statusMessage
+      )
+    }
+
+    return true
+  }
+
+  /**
+   * @see {DispatchHandlers.onData}
+   *
+   * @param {Buffer} chunk
+   * @returns {boolean}
+   */
+  onData (chunk) {
+    if (this.#successful) {
+      return true
+    }
+
+    if (typeof this.#handler.onData === 'function') {
+      return this.#handler.onData(chunk)
+    }
+
+    return true
+  }
+
+  /**
+   * @see {DispatchHandlers.onComplete}
+   *
+   * @param {string[] | null} rawTrailers
+   */
+  onComplete (rawTrailers) {
+    if (this.#successful) {
+      return
+    }
+
+    if (typeof this.#handler.onComplete === 'function') {
+      this.#handler.onComplete(rawTrailers)
+    }
+  }
+
+  /**
+   * @see {DispatchHandlers.onError}
+   *
+   * @param {Error} err
+   */
+  onError (err) {
+    if (this.#successful) {
+      return
+    }
+
+    if (this.#callback) {
+      this.#callback(false)
+      this.#callback = null
+    }
+
+    if (typeof this.#handler.onError === 'function') {
+      this.#handler.onError(err)
+    } else {
+      throw err
+    }
+  }
+}
+
+module.exports = CacheRevalidationHandler

package/lib/handler/redirect-handler.js

@@ -75,7 +75,8 @@ class RedirectHandler {
       this.opts.body &&
       typeof this.opts.body !== 'string' &&
       !ArrayBuffer.isView(this.opts.body) &&
-      util.isIterable(this.opts.body)
+      util.isIterable(this.opts.body) &&
+      !util.isFormDataLike(this.opts.body)
     ) {
       // TODO: Should we allow re-using iterable if !this.opts.idempotent
       // or through some other flag?
@@ -227,9 +228,10 @@ function cleanRequestHeaders (headers, removeContent, unknownOrigin) {
       }
     }
   } else if (headers && typeof headers === 'object') {
-    for (const key of Object.keys(headers)) {
+    const entries = typeof headers[Symbol.iterator] === 'function' ? headers : Object.entries(headers)
+    for (const [key, value] of entries) {
       if (!shouldRemoveHeader(key, removeContent, unknownOrigin)) {
-        ret.push(key, headers[key])
+        ret.push(key, value)
       }
     }
   } else {

package/lib/handler/retry-handler.js

@@ -229,7 +229,7 @@ class RetryHandler {
         return false
       }
 
-      const { start, size, end = size } = contentRange
+      const { start, size, end = size - 1 } = contentRange
 
       assert(this.start === start, 'content-range mismatch')
       assert(this.end == null || this.end === end, 'content-range mismatch')
@@ -252,7 +252,7 @@ class RetryHandler {
           )
         }
 
-        const { start, size, end = size } = range
+        const { start, size, end = size - 1 } = range
         assert(
           start != null && Number.isFinite(start),
           'content-range mismatch'
@@ -266,7 +266,7 @@ class RetryHandler {
       // We make our best to checkpoint the body for further range headers
       if (this.end == null) {
         const contentLength = headers['content-length']
-        this.end = contentLength != null ? Number(contentLength) : null
+        this.end = contentLength != null ? Number(contentLength) - 1 : null
       }
 
       assert(Number.isFinite(this.start))