undici 7.0.0-alpha.2 → 7.0.0-alpha.4

package/lib/interceptor/cache.js

@@ -0,0 +1,192 @@
+'use strict'
+
+const assert = require('node:assert')
+const { Readable } = require('node:stream')
+const util = require('../core/util')
+const CacheHandler = require('../handler/cache-handler')
+const MemoryCacheStore = require('../cache/memory-cache-store')
+const CacheRevalidationHandler = require('../handler/cache-revalidation-handler')
+const { assertCacheStore, assertCacheMethods, makeCacheKey } = require('../util/cache.js')
+
+const AGE_HEADER = Buffer.from('age')
+
+/**
+ * @typedef {import('../../types/cache-interceptor.d.ts').default.CachedResponse} CachedResponse
+ */
+
+/**
+ * @param {import('../../types/cache-interceptor.d.ts').default.CacheOptions} [opts]
+ * @returns {import('../../types/dispatcher.d.ts').default.DispatcherComposeInterceptor}
+ */
+module.exports = (opts = {}) => {
+  const {
+    store = new MemoryCacheStore(),
+    methods = ['GET']
+  } = opts
+
+  if (typeof opts !== 'object' || opts === null) {
+    throw new TypeError(`expected type of opts to be an Object, got ${opts === null ? 'null' : typeof opts}`)
+  }
+
+  assertCacheStore(store, 'opts.store')
+  assertCacheMethods(methods, 'opts.methods')
+
+  const globalOpts = {
+    store,
+    methods
+  }
+
+  const safeMethodsToNotCache = util.safeHTTPMethods.filter(method => methods.includes(method) === false)
+
+  return dispatch => {
+    return (opts, handler) => {
+      // TODO (fix): What if e.g. opts.headers has if-modified-since header? Or other headers
+      // that make things ambigious?
+
+      if (!opts.origin || safeMethodsToNotCache.includes(opts.method)) {
+        // Not a method we want to cache or we don't have the origin, skip
+        return dispatch(opts, handler)
+      }
+
+      /**
+       * @type {import('../../types/cache-interceptor.d.ts').default.CacheKey}
+       */
+      const cacheKey = makeCacheKey(opts)
+
+      // TODO (perf): For small entries support returning a Buffer instead of a stream.
+      // Maybe store should return { staleAt, headers, body, etc... } instead of a stream + stream.value?
+      // Where body can be a Buffer, string, stream or blob?
+      const result = store.get(cacheKey)
+      if (!result) {
+        return dispatch(opts, new CacheHandler(globalOpts, cacheKey, handler))
+      }
+
+      /**
+       * @param {import('../../types/cache-interceptor.d.ts').default.GetResult} result
+       */
+      const respondWithCachedValue = ({ cachedAt, rawHeaders, statusCode, statusMessage, body }) => {
+        const stream = util.isStream(body)
+          ? body
+          : Readable.from(body ?? [])
+
+        assert(!stream.destroyed, 'stream should not be destroyed')
+        assert(!stream.readableDidRead, 'stream should not be readableDidRead')
+
+        stream
+          .on('error', function (err) {
+            if (!this.readableEnded) {
+              if (typeof handler.onError === 'function') {
+                handler.onError(err)
+              } else {
+                throw err
+              }
+            }
+          })
+          .on('close', function () {
+            if (!this.errored && typeof handler.onComplete === 'function') {
+              handler.onComplete([])
+            }
+          })
+
+        if (typeof handler.onConnect === 'function') {
+          handler.onConnect((err) => {
+            stream.destroy(err)
+          })
+
+          if (stream.destroyed) {
+            return
+          }
+        }
+
+        if (typeof handler.onHeaders === 'function') {
+          // Add the age header
+          // https://www.rfc-editor.org/rfc/rfc9111.html#name-age
+          const age = Math.round((Date.now() - cachedAt) / 1000)
+
+          // TODO (fix): What if rawHeaders already contains age header?
+          rawHeaders = [...rawHeaders, AGE_HEADER, Buffer.from(`${age}`)]
+
+          if (handler.onHeaders(statusCode, rawHeaders, () => stream?.resume(), statusMessage) === false) {
+            stream.pause()
+          }
+        }
+
+        if (opts.method === 'HEAD') {
+          stream.destroy()
+        } else {
+          stream.on('data', function (chunk) {
+            if (typeof handler.onData === 'function' && !handler.onData(chunk)) {
+              stream.pause()
+            }
+          })
+        }
+      }
+
+      /**
+       * @param {import('../../types/cache-interceptor.d.ts').default.GetResult} result
+       */
+      const handleResult = (result) => {
+        // TODO (perf): Readable.from path can be optimized...
+
+        if (!result.body && opts.method !== 'HEAD') {
+          throw new Error('stream is undefined but method isn\'t HEAD')
+        }
+
+        // Check if the response is stale
+        const now = Date.now()
+        if (now < result.staleAt) {
+          // Dump request body.
+          if (util.isStream(opts.body)) {
+            opts.body.on('error', () => {}).destroy()
+          }
+          respondWithCachedValue(result)
+        } else if (util.isStream(opts.body) && util.bodyLength(opts.body) !== 0) {
+          // If body is is stream we can't revalidate...
+          // TODO (fix): This could be less strict...
+          dispatch(opts, new CacheHandler(globalOpts, cacheKey, handler))
+        } else {
+          // Need to revalidate the response
+          dispatch(
+            {
+              ...opts,
+              headers: {
+                ...opts.headers,
+                'if-modified-since': new Date(result.cachedAt).toUTCString()
+              }
+            },
+            new CacheRevalidationHandler(
+              (success) => {
+                if (success) {
+                  respondWithCachedValue(result)
+                } else if (util.isStream(result.body)) {
+                  result.body.on('error', () => {}).destroy()
+                }
+              },
+              new CacheHandler(globalOpts, cacheKey, handler)
+            )
+          )
+        }
+      }
+
+      if (typeof result.then === 'function') {
+        result.then((result) => {
+          if (!result) {
+            dispatch(opts, new CacheHandler(globalOpts, cacheKey, handler))
+          } else {
+            handleResult(result)
+          }
+        }, err => {
+          if (typeof handler.onError === 'function') {
+            handler.onError(err)
+          } else {
+            throw err
+          }
+        })
+      } else {
+        handleResult(result)
+      }
+
+      return true
+    }
+  }
+}

package/lib/interceptor/dns.js

@@ -13,7 +13,6 @@ class DNSInstance {
   affinity = null
   lookup = null
   pick = null
-  lastIpFamily = null
 
   constructor (opts) {
     this.#maxTTL = opts.maxTTL
@@ -61,16 +60,23 @@ class DNSInstance {
         const ip = this.pick(
           origin,
           records,
-          // Only set affinity if dual stack is disabled
-          // otherwise let it go through normal flow
-          !newOpts.dualStack && newOpts.affinity
+          newOpts.affinity
         )
 
+        let port
+        if (typeof ip.port === 'number') {
+          port = `:${ip.port}`
+        } else if (origin.port !== '') {
+          port = `:${origin.port}`
+        } else {
+          port = ''
+        }
+
         cb(
           null,
           `${origin.protocol}//${
             ip.family === 6 ? `[${ip.address}]` : ip.address
-          }${origin.port === '' ? '' : `:${origin.port}`}`
+          }${port}`
         )
       })
     } else {
@@ -78,9 +84,7 @@ class DNSInstance {
       const ip = this.pick(
         origin,
         ips,
-        // Only set affinity if dual stack is disabled
-        // otherwise let it go through normal flow
-        !newOpts.dualStack && newOpts.affinity
+        newOpts.affinity
       )
 
       // If no IPs we lookup - deleting old records
@@ -90,11 +94,20 @@ class DNSInstance {
         return
       }
 
+      let port
+      if (typeof ip.port === 'number') {
+        port = `:${ip.port}`
+      } else if (origin.port !== '') {
+        port = `:${origin.port}`
+      } else {
+        port = ''
+      }
+
       cb(
         null,
         `${origin.protocol}//${
           ip.family === 6 ? `[${ip.address}]` : ip.address
-        }${origin.port === '' ? '' : `:${origin.port}`}`
+        }${port}`
       )
     }
   }
@@ -102,7 +115,11 @@ class DNSInstance {
   #defaultLookup (origin, opts, cb) {
     lookup(
       origin.hostname,
-      { all: true, family: this.dualStack === false ? this.affinity : 0 },
+      {
+        all: true,
+        family: this.dualStack === false ? this.affinity : 0,
+        order: 'ipv4first'
+      },
       (err, addresses) => {
         if (err) {
           return cb(err)
@@ -111,15 +128,9 @@ class DNSInstance {
         const results = new Map()
 
         for (const addr of addresses) {
-          const record = {
-            address: addr.address,
-            ttl: opts.maxTTL,
-            family: addr.family
-          }
-
           // On linux we found duplicates, we attempt to remove them with
           // the latest record
-          results.set(`${record.address}:${record.family}`, record)
+          results.set(`${addr.address}:${addr.family}`, addr)
         }
 
         cb(null, results.values())
@@ -129,36 +140,36 @@ class DNSInstance {
 
   #defaultPick (origin, hostnameRecords, affinity) {
     let ip = null
-    const { records, offset = 0 } = hostnameRecords
-    let newOffset = 0
+    const { records, offset } = hostnameRecords
+
+    let family
+    if (this.dualStack) {
+      if (affinity == null) {
+        // Balance between ip families
+        if (offset == null || offset === maxInt) {
+          hostnameRecords.offset = 0
+          affinity = 4
+        } else {
+          hostnameRecords.offset++
+          affinity = (hostnameRecords.offset & 1) === 1 ? 6 : 4
+        }
+      }
 
-    if (offset === maxInt) {
-      newOffset = 0
+      if (records[affinity] != null && records[affinity].ips.length > 0) {
+        family = records[affinity]
+      } else {
+        family = records[affinity === 4 ? 6 : 4]
+      }
     } else {
-      newOffset = offset + 1
+      family = records[affinity]
     }
 
-    // We balance between the two IP families
-    // If dual-stack disabled, we automatically pick the affinity
-    const newIpFamily = (newOffset & 1) === 1 ? 4 : 6
-    const family =
-      this.dualStack === false
-        ? records[this.affinity] // If dual-stack is disabled, we pick the default affiniy
-        : records[affinity] ?? records[newIpFamily]
-
-    // If no IPs and we have tried both families or dual stack is disabled, we return null
-    if (
-      (family == null || family.ips.length === 0) &&
-      // eslint-disable-next-line eqeqeq
-      (this.dualStack === false || this.lastIpFamily != newIpFamily)
-    ) {
+    // If no IPs we return null
+    if (family == null || family.ips.length === 0) {
       return ip
     }
 
-    family.offset = family.offset ?? 0
-    hostnameRecords.offset = newOffset
-
-    if (family.offset === maxInt) {
+    if (family.offset == null || family.offset === maxInt) {
       family.offset = 0
     } else {
       family.offset++
@@ -171,24 +182,28 @@ class DNSInstance {
       return ip
     }
 
-    const timestamp = Date.now()
-    // Record TTL is already in ms
-    if (ip.timestamp != null && timestamp - ip.timestamp > ip.ttl) {
+    if (Date.now() - ip.timestamp > ip.ttl) { // record TTL is already in ms
       // We delete expired records
       // It is possible that they have different TTL, so we manage them individually
       family.ips.splice(position, 1)
       return this.pick(origin, hostnameRecords, affinity)
     }
 
-    ip.timestamp = timestamp
-
-    this.lastIpFamily = newIpFamily
     return ip
   }
 
   setRecords (origin, addresses) {
+    const timestamp = Date.now()
     const records = { records: { 4: null, 6: null } }
     for (const record of addresses) {
+      record.timestamp = timestamp
+      if (typeof record.ttl === 'number') {
+        // The record TTL is expected to be in ms
+        record.ttl = Math.min(record.ttl, this.#maxTTL)
+      } else {
+        record.ttl = this.#maxTTL
+      }
+
       const familyRecords = records.records[record.family] ?? { ips: [] }
 
       familyRecords.ips.push(record)
@@ -302,12 +317,20 @@ module.exports = interceptorOpts => {
     throw new InvalidArgumentError('Invalid pick. Must be a function')
   }
 
+  const dualStack = interceptorOpts?.dualStack ?? true
+  let affinity
+  if (dualStack) {
+    affinity = interceptorOpts?.affinity ?? null
+  } else {
+    affinity = interceptorOpts?.affinity ?? 4
+  }
+
   const opts = {
     maxTTL: interceptorOpts?.maxTTL ?? 10e3, // Expressed in ms
     lookup: interceptorOpts?.lookup ?? null,
     pick: interceptorOpts?.pick ?? null,
-    dualStack: interceptorOpts?.dualStack ?? true,
-    affinity: interceptorOpts?.affinity ?? 4,
+    dualStack,
+    affinity,
     maxItems: interceptorOpts?.maxItems ?? Infinity
   }
 

package/lib/util/cache.js

@@ -0,0 +1,249 @@
+'use strict'
+
+const {
+  safeHTTPMethods
+} = require('../core/util')
+
+/**
+ *
+ * @param {import('../../types/dispatcher.d.ts').default.DispatchOptions} opts
+ */
+function makeCacheKey (opts) {
+  if (!opts.origin) {
+    throw new Error('opts.origin is undefined')
+  }
+
+  /**
+   * @type {import('../../types/cache-interceptor.d.ts').default.CacheKey}
+   */
+  const cacheKey = {
+    origin: opts.origin.toString(),
+    method: opts.method,
+    path: opts.path,
+    headers: opts.headers
+  }
+
+  return cacheKey
+}
+
+/**
+ * @see https://www.rfc-editor.org/rfc/rfc9111.html#name-cache-control
+ * @see https://www.iana.org/assignments/http-cache-directives/http-cache-directives.xhtml
+ *
+ * @typedef {{
+ *   'max-stale'?: number;
+ *   'min-fresh'?: number;
+ *   'max-age'?: number;
+ *   's-maxage'?: number;
+ *   'stale-while-revalidate'?: number;
+ *   'stale-if-error'?: number;
+ *   public?: true;
+ *   private?: true | string[];
+ *   'no-store'?: true;
+ *   'no-cache'?: true | string[];
+ *   'must-revalidate'?: true;
+ *   'proxy-revalidate'?: true;
+ *   immutable?: true;
+ *   'no-transform'?: true;
+ *   'must-understand'?: true;
+ *   'only-if-cached'?: true;
+ * }} CacheControlDirectives
+ *
+ * @param {string | string[]} header
+ * @returns {CacheControlDirectives}
+ */
+function parseCacheControlHeader (header) {
+  /**
+   * @type {import('../util/cache.js').CacheControlDirectives}
+   */
+  const output = {}
+
+  const directives = Array.isArray(header) ? header : header.split(',')
+  for (let i = 0; i < directives.length; i++) {
+    const directive = directives[i].toLowerCase()
+    const keyValueDelimiter = directive.indexOf('=')
+
+    let key
+    let value
+    if (keyValueDelimiter !== -1) {
+      key = directive.substring(0, keyValueDelimiter).trim()
+      value = directive
+        .substring(keyValueDelimiter + 1)
+        .trim()
+    } else {
+      key = directive.trim()
+    }
+
+    switch (key) {
+      case 'min-fresh':
+      case 'max-stale':
+      case 'max-age':
+      case 's-maxage':
+      case 'stale-while-revalidate':
+      case 'stale-if-error': {
+        if (value === undefined) {
+          continue
+        }
+
+        const parsedValue = parseInt(value, 10)
+        // eslint-disable-next-line no-self-compare
+        if (parsedValue !== parsedValue) {
+          continue
+        }
+
+        output[key] = parsedValue
+
+        break
+      }
+      case 'private':
+      case 'no-cache': {
+        if (value) {
+          // The private and no-cache directives can be unqualified (aka just
+          //  `private` or `no-cache`) or qualified (w/ a value). When they're
+          //  qualified, it's a list of headers like `no-cache=header1`,
+          //  `no-cache="header1"`, or `no-cache="header1, header2"`
+          // If we're given multiple headers, the comma messes us up since
+          //  we split the full header by commas. So, let's loop through the
+          //  remaining parts in front of us until we find one that ends in a
+          //  quote. We can then just splice all of the parts in between the
+          //  starting quote and the ending quote out of the directives array
+          //  and continue parsing like normal.
+          // https://www.rfc-editor.org/rfc/rfc9111.html#name-no-cache-2
+          if (value[0] === '"') {
+            // Something like `no-cache="some-header"` OR `no-cache="some-header, another-header"`.
+
+            // Add the first header on and cut off the leading quote
+            const headers = [value.substring(1)]
+
+            let foundEndingQuote = value[value.length - 1] === '"'
+            if (!foundEndingQuote) {
+              // Something like `no-cache="some-header, another-header"`
+              //  This can still be something invalid, e.g. `no-cache="some-header, ...`
+              for (let j = i + 1; j < directives.length; j++) {
+                const nextPart = directives[j]
+                const nextPartLength = nextPart.length
+
+                headers.push(nextPart.trim())
+
+                if (nextPartLength !== 0 && nextPart[nextPartLength - 1] === '"') {
+                  foundEndingQuote = true
+                  break
+                }
+              }
+            }
+
+            if (foundEndingQuote) {
+              let lastHeader = headers[headers.length - 1]
+              if (lastHeader[lastHeader.length - 1] === '"') {
+                lastHeader = lastHeader.substring(0, lastHeader.length - 1)
+                headers[headers.length - 1] = lastHeader
+              }
+
+              output[key] = headers
+            }
+          } else {
+            // Something like `no-cache=some-header`
+            output[key] = [value]
+          }
+
+          break
+        }
+      }
+      // eslint-disable-next-line no-fallthrough
+      case 'public':
+      case 'no-store':
+      case 'must-revalidate':
+      case 'proxy-revalidate':
+      case 'immutable':
+      case 'no-transform':
+      case 'must-understand':
+      case 'only-if-cached':
+        if (value) {
+          // These are qualified (something like `public=...`) when they aren't
+          //  allowed to be, skip
+          continue
+        }
+
+        output[key] = true
+        break
+      default:
+        // Ignore unknown directives as per https://www.rfc-editor.org/rfc/rfc9111.html#section-5.2.3-1
+        continue
+    }
+  }
+
+  return output
+}
+
+/**
+ * @param {string | string[]} varyHeader Vary header from the server
+ * @param {Record<string, string | string[]>} headers Request headers
+ * @returns {Record<string, string | string[]>}
+ */
+function parseVaryHeader (varyHeader, headers) {
+  if (typeof varyHeader === 'string' && varyHeader === '*') {
+    return headers
+  }
+
+  const output = /** @type {Record<string, string | string[]>} */ ({})
+
+  const varyingHeaders = typeof varyHeader === 'string'
+    ? varyHeader.split(',')
+    : varyHeader
+  for (const header of varyingHeaders) {
+    const trimmedHeader = header.trim().toLowerCase()
+
+    if (headers[trimmedHeader]) {
+      output[trimmedHeader] = headers[trimmedHeader]
+    }
+  }
+
+  return output
+}
+
+/**
+ * @param {unknown} store
+ * @returns {asserts store is import('../../types/cache-interceptor.d.ts').default.CacheStore}
+ */
+function assertCacheStore (store, name = 'CacheStore') {
+  if (typeof store !== 'object' || store === null) {
+    throw new TypeError(`expected type of ${name} to be a CacheStore, got ${store === null ? 'null' : typeof store}`)
+  }
+
+  for (const fn of ['get', 'createWriteStream', 'delete']) {
+    if (typeof store[fn] !== 'function') {
+      throw new TypeError(`${name} needs to have a \`${fn}()\` function`)
+    }
+  }
+
+  if (typeof store.isFull !== 'undefined' && typeof store.isFull !== 'boolean') {
+    throw new TypeError(`${name} needs a isFull getter with type boolean or undefined, current type: ${typeof store.isFull}`)
+  }
+}
+/**
+ * @param {unknown} methods
+ * @returns {asserts methods is import('../../types/cache-interceptor.d.ts').default.CacheMethods[]}
+ */
+function assertCacheMethods (methods, name = 'CacheMethods') {
+  if (!Array.isArray(methods)) {
+    throw new TypeError(`expected type of ${name} needs to be an array, got ${methods === null ? 'null' : typeof methods}`)
+  }
+
+  if (methods.length === 0) {
+    throw new TypeError(`${name} needs to have at least one method`)
+  }
+
+  for (const method of methods) {
+    if (!safeHTTPMethods.includes(method)) {
+      throw new TypeError(`element of ${name}-array needs to be one of following values: ${safeHTTPMethods.join(', ')}, got ${method}`)
+    }
+  }
+}
+
+module.exports = {
+  makeCacheKey,
+  parseCacheControlHeader,
+  parseVaryHeader,
+  assertCacheMethods,
+  assertCacheStore
+}

package/lib/web/cache/cache.js

@@ -36,6 +36,7 @@ class Cache {
       webidl.illegalConstructor()
     }
 
+    webidl.util.markAsUncloneable(this)
     this.#relevantRequestResponseList = arguments[1]
   }
 

package/lib/web/cache/cachestorage.js

@@ -16,6 +16,8 @@ class CacheStorage {
     if (arguments[0] !== kConstruct) {
       webidl.illegalConstructor()
     }
+
+    webidl.util.markAsUncloneable(this)
   }
 
   async match (request, options = {}) {

package/lib/web/cookies/index.js

@@ -91,6 +91,16 @@ function getSetCookies (headers) {
   return cookies.map((pair) => parseSetCookie(pair))
 }
 
+/**
+ * Parses a cookie string
+ * @param {string} cookie
+ */
+function parseCookie (cookie) {
+  cookie = webidl.converters.DOMString(cookie)
+
+  return parseSetCookie(cookie)
+}
+
 /**
  * @param {Headers} headers
  * @param {Cookie} cookie
@@ -184,5 +194,6 @@ module.exports = {
   getCookies,
   deleteCookie,
   getSetCookies,
-  setCookie
+  setCookie,
+  parseCookie
 }