undici 7.0.0-alpha.3 → 7.0.0-alpha.4

package/lib/handler/retry-handler.js

@@ -229,7 +229,7 @@ class RetryHandler {
         return false
       }
 
-      const { start, size, end = size } = contentRange
+      const { start, size, end = size - 1 } = contentRange
 
       assert(this.start === start, 'content-range mismatch')
       assert(this.end == null || this.end === end, 'content-range mismatch')
@@ -252,7 +252,7 @@ class RetryHandler {
           )
         }
 
-        const { start, size, end = size } = range
+        const { start, size, end = size - 1 } = range
         assert(
           start != null && Number.isFinite(start),
           'content-range mismatch'
@@ -266,7 +266,7 @@ class RetryHandler {
       // We make our best to checkpoint the body for further range headers
       if (this.end == null) {
         const contentLength = headers['content-length']
-        this.end = contentLength != null ? Number(contentLength) : null
+        this.end = contentLength != null ? Number(contentLength) - 1 : null
       }
 
       assert(Number.isFinite(this.start))

package/lib/interceptor/cache.js

@@ -1,13 +1,19 @@
 'use strict'
 
+const assert = require('node:assert')
+const { Readable } = require('node:stream')
 const util = require('../core/util')
 const CacheHandler = require('../handler/cache-handler')
 const MemoryCacheStore = require('../cache/memory-cache-store')
 const CacheRevalidationHandler = require('../handler/cache-revalidation-handler')
-const { assertCacheStore, assertCacheMethods } = require('../util/cache.js')
+const { assertCacheStore, assertCacheMethods, makeCacheKey } = require('../util/cache.js')
 
 const AGE_HEADER = Buffer.from('age')
 
+/**
+ * @typedef {import('../../types/cache-interceptor.d.ts').default.CachedResponse} CachedResponse
+ */
+
 /**
  * @param {import('../../types/cache-interceptor.d.ts').default.CacheOptions} [opts]
  * @returns {import('../../types/dispatcher.d.ts').default.DispatcherComposeInterceptor}
@@ -34,136 +40,151 @@ module.exports = (opts = {}) => {
 
   return dispatch => {
     return (opts, handler) => {
+      // TODO (fix): What if e.g. opts.headers has if-modified-since header? Or other headers
+      // that make things ambigious?
+
       if (!opts.origin || safeMethodsToNotCache.includes(opts.method)) {
         // Not a method we want to cache or we don't have the origin, skip
         return dispatch(opts, handler)
       }
 
-      const stream = store.createReadStream(opts)
-      if (!stream) {
-        // Request isn't cached
-        return dispatch(opts, new CacheHandler(globalOpts, opts, handler))
+      /**
+       * @type {import('../../types/cache-interceptor.d.ts').default.CacheKey}
+       */
+      const cacheKey = makeCacheKey(opts)
+
+      // TODO (perf): For small entries support returning a Buffer instead of a stream.
+      // Maybe store should return { staleAt, headers, body, etc... } instead of a stream + stream.value?
+      // Where body can be a Buffer, string, stream or blob?
+      const result = store.get(cacheKey)
+      if (!result) {
+        return dispatch(opts, new CacheHandler(globalOpts, cacheKey, handler))
       }
 
-      let onErrorCalled = false
-
       /**
-       * @param {import('../../types/cache-interceptor.d.ts').default.CacheStoreReadable} stream
-       * @param {import('../../types/cache-interceptor.d.ts').default.CacheStoreValue} value
+       * @param {import('../../types/cache-interceptor.d.ts').default.GetResult} result
        */
-      const respondWithCachedValue = (stream, value) => {
-        const ac = new AbortController()
-        const signal = ac.signal
-
-        signal.onabort = (_, err) => {
-          stream.destroy()
-          if (!onErrorCalled) {
-            handler.onError(err)
-            onErrorCalled = true
-          }
-        }
+      const respondWithCachedValue = ({ cachedAt, rawHeaders, statusCode, statusMessage, body }) => {
+        const stream = util.isStream(body)
+          ? body
+          : Readable.from(body ?? [])
+
+        assert(!stream.destroyed, 'stream should not be destroyed')
+        assert(!stream.readableDidRead, 'stream should not be readableDidRead')
+
+        stream
+          .on('error', function (err) {
+            if (!this.readableEnded) {
+              if (typeof handler.onError === 'function') {
+                handler.onError(err)
+              } else {
+                throw err
+              }
+            }
+          })
+          .on('close', function () {
+            if (!this.errored && typeof handler.onComplete === 'function') {
+              handler.onComplete([])
+            }
+          })
 
-        stream.on('error', (err) => {
-          if (!onErrorCalled) {
-            handler.onError(err)
-            onErrorCalled = true
-          }
-        })
+        if (typeof handler.onConnect === 'function') {
+          handler.onConnect((err) => {
+            stream.destroy(err)
+          })
 
-        try {
-          if (typeof handler.onConnect === 'function') {
-            handler.onConnect(ac.abort)
-            signal.throwIfAborted()
+          if (stream.destroyed) {
+            return
           }
+        }
 
-          if (typeof handler.onHeaders === 'function') {
-            // Add the age header
-            // https://www.rfc-editor.org/rfc/rfc9111.html#name-age
-            const age = Math.round((Date.now() - value.cachedAt) / 1000)
+        if (typeof handler.onHeaders === 'function') {
+          // Add the age header
+          // https://www.rfc-editor.org/rfc/rfc9111.html#name-age
+          const age = Math.round((Date.now() - cachedAt) / 1000)
 
-            value.rawHeaders.push(AGE_HEADER, Buffer.from(`${age}`))
+          // TODO (fix): What if rawHeaders already contains age header?
+          rawHeaders = [...rawHeaders, AGE_HEADER, Buffer.from(`${age}`)]
 
-            handler.onHeaders(value.statusCode, value.rawHeaders, stream.resume, value.statusMessage)
-            signal.throwIfAborted()
+          if (handler.onHeaders(statusCode, rawHeaders, () => stream?.resume(), statusMessage) === false) {
+            stream.pause()
           }
+        }
 
-          if (opts.method === 'HEAD') {
-            if (typeof handler.onComplete === 'function') {
-              handler.onComplete(null)
-              stream.destroy()
-            }
-          } else {
-            if (typeof handler.onData === 'function') {
-              stream.on('data', chunk => {
-                if (!handler.onData(chunk)) {
-                  stream.pause()
-                }
-              })
-            }
-
-            if (typeof handler.onComplete === 'function') {
-              stream.on('end', () => {
-                handler.onComplete(value.rawTrailers ?? [])
-              })
+        if (opts.method === 'HEAD') {
+          stream.destroy()
+        } else {
+          stream.on('data', function (chunk) {
+            if (typeof handler.onData === 'function' && !handler.onData(chunk)) {
+              stream.pause()
             }
-          }
-        } catch (err) {
-          stream.destroy(err)
-          if (!onErrorCalled && typeof handler.onError === 'function') {
-            handler.onError(err)
-            onErrorCalled = true
-          }
+          })
         }
       }
 
       /**
-       * @param {import('../../types/cache-interceptor.d.ts').default.CacheStoreReadable | undefined} stream
+       * @param {import('../../types/cache-interceptor.d.ts').default.GetResult} result
        */
-      const handleStream = (stream) => {
-        if (!stream) {
-          // Request isn't cached
-          return dispatch(opts, new CacheHandler(globalOpts, opts, handler))
-        }
-
-        const { value } = stream
+      const handleResult = (result) => {
+        // TODO (perf): Readable.from path can be optimized...
 
-        // Dump body on error
-        if (util.isStream(opts.body)) {
-          opts.body?.on('error', () => {}).resume()
+        if (!result.body && opts.method !== 'HEAD') {
+          throw new Error('stream is undefined but method isn\'t HEAD')
         }
 
         // Check if the response is stale
         const now = Date.now()
-        if (now >= value.staleAt) {
-          if (now >= value.deleteAt) {
-            // Safety check in case the store gave us a response that should've been
-            //  deleted already
-            dispatch(opts, new CacheHandler(globalOpts, opts, handler))
-            return
+        if (now < result.staleAt) {
+          // Dump request body.
+          if (util.isStream(opts.body)) {
+            opts.body.on('error', () => {}).destroy()
           }
-
-          if (!opts.headers) {
-            opts.headers = {}
-          }
-
-          opts.headers['if-modified-since'] = new Date(value.cachedAt).toUTCString()
-
+          respondWithCachedValue(result)
+        } else if (util.isStream(opts.body) && util.bodyLength(opts.body) !== 0) {
+          // If body is is stream we can't revalidate...
+          // TODO (fix): This could be less strict...
+          dispatch(opts, new CacheHandler(globalOpts, cacheKey, handler))
+        } else {
           // Need to revalidate the response
           dispatch(
-            opts,
+            {
+              ...opts,
+              headers: {
+                ...opts.headers,
+                'if-modified-since': new Date(result.cachedAt).toUTCString()
+              }
+            },
             new CacheRevalidationHandler(
-              () => respondWithCachedValue(stream, value),
-              new CacheHandler(globalOpts, opts, handler)
+              (success) => {
+                if (success) {
+                  respondWithCachedValue(result)
+                } else if (util.isStream(result.body)) {
+                  result.body.on('error', () => {}).destroy()
+                }
+              },
+              new CacheHandler(globalOpts, cacheKey, handler)
             )
           )
-
-          return
         }
-
-        respondWithCachedValue(stream, value)
       }
 
-      Promise.resolve(stream).then(handleStream).catch(handler.onError)
+      if (typeof result.then === 'function') {
+        result.then((result) => {
+          if (!result) {
+            dispatch(opts, new CacheHandler(globalOpts, cacheKey, handler))
+          } else {
+            handleResult(result)
+          }
+        }, err => {
+          if (typeof handler.onError === 'function') {
+            handler.onError(err)
+          } else {
+            throw err
+          }
+        })
+      } else {
+        handleResult(result)
+      }
 
       return true
     }

package/lib/interceptor/dns.js

@@ -13,7 +13,6 @@ class DNSInstance {
   affinity = null
   lookup = null
   pick = null
-  lastIpFamily = null
 
   constructor (opts) {
     this.#maxTTL = opts.maxTTL
@@ -61,16 +60,23 @@ class DNSInstance {
         const ip = this.pick(
           origin,
           records,
-          // Only set affinity if dual stack is disabled
-          // otherwise let it go through normal flow
-          !newOpts.dualStack && newOpts.affinity
+          newOpts.affinity
         )
 
+        let port
+        if (typeof ip.port === 'number') {
+          port = `:${ip.port}`
+        } else if (origin.port !== '') {
+          port = `:${origin.port}`
+        } else {
+          port = ''
+        }
+
         cb(
           null,
           `${origin.protocol}//${
             ip.family === 6 ? `[${ip.address}]` : ip.address
-          }${origin.port === '' ? '' : `:${origin.port}`}`
+          }${port}`
         )
       })
     } else {
@@ -78,9 +84,7 @@ class DNSInstance {
       const ip = this.pick(
         origin,
         ips,
-        // Only set affinity if dual stack is disabled
-        // otherwise let it go through normal flow
-        !newOpts.dualStack && newOpts.affinity
+        newOpts.affinity
       )
 
       // If no IPs we lookup - deleting old records
@@ -90,11 +94,20 @@ class DNSInstance {
         return
       }
 
+      let port
+      if (typeof ip.port === 'number') {
+        port = `:${ip.port}`
+      } else if (origin.port !== '') {
+        port = `:${origin.port}`
+      } else {
+        port = ''
+      }
+
       cb(
         null,
         `${origin.protocol}//${
           ip.family === 6 ? `[${ip.address}]` : ip.address
-        }${origin.port === '' ? '' : `:${origin.port}`}`
+        }${port}`
       )
     }
   }
@@ -102,7 +115,11 @@ class DNSInstance {
   #defaultLookup (origin, opts, cb) {
     lookup(
       origin.hostname,
-      { all: true, family: this.dualStack === false ? this.affinity : 0 },
+      {
+        all: true,
+        family: this.dualStack === false ? this.affinity : 0,
+        order: 'ipv4first'
+      },
       (err, addresses) => {
         if (err) {
           return cb(err)
@@ -111,15 +128,9 @@ class DNSInstance {
         const results = new Map()
 
         for (const addr of addresses) {
-          const record = {
-            address: addr.address,
-            ttl: opts.maxTTL,
-            family: addr.family
-          }
-
           // On linux we found duplicates, we attempt to remove them with
           // the latest record
-          results.set(`${record.address}:${record.family}`, record)
+          results.set(`${addr.address}:${addr.family}`, addr)
         }
 
         cb(null, results.values())
@@ -129,36 +140,36 @@ class DNSInstance {
 
   #defaultPick (origin, hostnameRecords, affinity) {
     let ip = null
-    const { records, offset = 0 } = hostnameRecords
-    let newOffset = 0
+    const { records, offset } = hostnameRecords
+
+    let family
+    if (this.dualStack) {
+      if (affinity == null) {
+        // Balance between ip families
+        if (offset == null || offset === maxInt) {
+          hostnameRecords.offset = 0
+          affinity = 4
+        } else {
+          hostnameRecords.offset++
+          affinity = (hostnameRecords.offset & 1) === 1 ? 6 : 4
+        }
+      }
 
-    if (offset === maxInt) {
-      newOffset = 0
+      if (records[affinity] != null && records[affinity].ips.length > 0) {
+        family = records[affinity]
+      } else {
+        family = records[affinity === 4 ? 6 : 4]
+      }
     } else {
-      newOffset = offset + 1
+      family = records[affinity]
     }
 
-    // We balance between the two IP families
-    // If dual-stack disabled, we automatically pick the affinity
-    const newIpFamily = (newOffset & 1) === 1 ? 4 : 6
-    const family =
-      this.dualStack === false
-        ? records[this.affinity] // If dual-stack is disabled, we pick the default affiniy
-        : records[affinity] ?? records[newIpFamily]
-
-    // If no IPs and we have tried both families or dual stack is disabled, we return null
-    if (
-      (family == null || family.ips.length === 0) &&
-      // eslint-disable-next-line eqeqeq
-      (this.dualStack === false || this.lastIpFamily != newIpFamily)
-    ) {
+    // If no IPs we return null
+    if (family == null || family.ips.length === 0) {
       return ip
     }
 
-    family.offset = family.offset ?? 0
-    hostnameRecords.offset = newOffset
-
-    if (family.offset === maxInt) {
+    if (family.offset == null || family.offset === maxInt) {
       family.offset = 0
     } else {
       family.offset++
@@ -171,24 +182,28 @@ class DNSInstance {
       return ip
     }
 
-    const timestamp = Date.now()
-    // Record TTL is already in ms
-    if (ip.timestamp != null && timestamp - ip.timestamp > ip.ttl) {
+    if (Date.now() - ip.timestamp > ip.ttl) { // record TTL is already in ms
       // We delete expired records
       // It is possible that they have different TTL, so we manage them individually
       family.ips.splice(position, 1)
       return this.pick(origin, hostnameRecords, affinity)
     }
 
-    ip.timestamp = timestamp
-
-    this.lastIpFamily = newIpFamily
     return ip
   }
 
   setRecords (origin, addresses) {
+    const timestamp = Date.now()
     const records = { records: { 4: null, 6: null } }
     for (const record of addresses) {
+      record.timestamp = timestamp
+      if (typeof record.ttl === 'number') {
+        // The record TTL is expected to be in ms
+        record.ttl = Math.min(record.ttl, this.#maxTTL)
+      } else {
+        record.ttl = this.#maxTTL
+      }
+
       const familyRecords = records.records[record.family] ?? { ips: [] }
 
       familyRecords.ips.push(record)
@@ -302,12 +317,20 @@ module.exports = interceptorOpts => {
     throw new InvalidArgumentError('Invalid pick. Must be a function')
   }
 
+  const dualStack = interceptorOpts?.dualStack ?? true
+  let affinity
+  if (dualStack) {
+    affinity = interceptorOpts?.affinity ?? null
+  } else {
+    affinity = interceptorOpts?.affinity ?? 4
+  }
+
   const opts = {
     maxTTL: interceptorOpts?.maxTTL ?? 10e3, // Expressed in ms
     lookup: interceptorOpts?.lookup ?? null,
     pick: interceptorOpts?.pick ?? null,
-    dualStack: interceptorOpts?.dualStack ?? true,
-    affinity: interceptorOpts?.affinity ?? 4,
+    dualStack,
+    affinity,
     maxItems: interceptorOpts?.maxItems ?? Infinity
   }
 

package/lib/util/cache.js

@@ -4,6 +4,28 @@ const {
   safeHTTPMethods
 } = require('../core/util')
 
+/**
+ *
+ * @param {import('../../types/dispatcher.d.ts').default.DispatchOptions} opts
+ */
+function makeCacheKey (opts) {
+  if (!opts.origin) {
+    throw new Error('opts.origin is undefined')
+  }
+
+  /**
+   * @type {import('../../types/cache-interceptor.d.ts').default.CacheKey}
+   */
+  const cacheKey = {
+    origin: opts.origin.toString(),
+    method: opts.method,
+    path: opts.path,
+    headers: opts.headers
+  }
+
+  return cacheKey
+}
+
 /**
  * @see https://www.rfc-editor.org/rfc/rfc9111.html#name-cache-control
  * @see https://www.iana.org/assignments/http-cache-directives/http-cache-directives.xhtml
@@ -27,7 +49,7 @@ const {
  *   'only-if-cached'?: true;
  * }} CacheControlDirectives
  *
- * @param {string} header
+ * @param {string | string[]} header
  * @returns {CacheControlDirectives}
  */
 function parseCacheControlHeader (header) {
@@ -36,9 +58,9 @@ function parseCacheControlHeader (header) {
    */
   const output = {}
 
-  const directives = header.toLowerCase().split(',')
+  const directives = Array.isArray(header) ? header : header.split(',')
   for (let i = 0; i < directives.length; i++) {
-    const directive = directives[i]
+    const directive = directives[i].toLowerCase()
     const keyValueDelimiter = directive.indexOf('=')
 
     let key
@@ -154,20 +176,22 @@ function parseCacheControlHeader (header) {
 }
 
 /**
- * @param {string} varyHeader Vary header from the server
- * @param {Record<string, string>} headers Request headers
- * @returns {Record<string, string>}
+ * @param {string | string[]} varyHeader Vary header from the server
+ * @param {Record<string, string | string[]>} headers Request headers
+ * @returns {Record<string, string | string[]>}
  */
 function parseVaryHeader (varyHeader, headers) {
-  if (varyHeader === '*') {
+  if (typeof varyHeader === 'string' && varyHeader === '*') {
     return headers
   }
 
-  const output = /** @type {Record<string, string>} */ ({})
+  const output = /** @type {Record<string, string | string[]>} */ ({})
 
-  const varyingHeaders = varyHeader.toLowerCase().split(',')
+  const varyingHeaders = typeof varyHeader === 'string'
+    ? varyHeader.split(',')
+    : varyHeader
   for (const header of varyingHeaders) {
-    const trimmedHeader = header.trim()
+    const trimmedHeader = header.trim().toLowerCase()
 
     if (headers[trimmedHeader]) {
       output[trimmedHeader] = headers[trimmedHeader]
@@ -186,14 +210,14 @@ function assertCacheStore (store, name = 'CacheStore') {
     throw new TypeError(`expected type of ${name} to be a CacheStore, got ${store === null ? 'null' : typeof store}`)
   }
 
-  for (const fn of ['createReadStream', 'createWriteStream', 'deleteByOrigin']) {
+  for (const fn of ['get', 'createWriteStream', 'delete']) {
     if (typeof store[fn] !== 'function') {
       throw new TypeError(`${name} needs to have a \`${fn}()\` function`)
     }
   }
 
-  if (typeof store.isFull !== 'boolean') {
-    throw new TypeError(`${name} needs a isFull getter with type boolean, current type: ${typeof store.isFull}`)
+  if (typeof store.isFull !== 'undefined' && typeof store.isFull !== 'boolean') {
+    throw new TypeError(`${name} needs a isFull getter with type boolean or undefined, current type: ${typeof store.isFull}`)
   }
 }
 /**
@@ -217,6 +241,7 @@ function assertCacheMethods (methods, name = 'CacheMethods') {
 }
 
 module.exports = {
+  makeCacheKey,
   parseCacheControlHeader,
   parseVaryHeader,
   assertCacheMethods,

package/lib/web/cookies/index.js

@@ -91,6 +91,16 @@ function getSetCookies (headers) {
   return cookies.map((pair) => parseSetCookie(pair))
 }
 
+/**
+ * Parses a cookie string
+ * @param {string} cookie
+ */
+function parseCookie (cookie) {
+  cookie = webidl.converters.DOMString(cookie)
+
+  return parseSetCookie(cookie)
+}
+
 /**
  * @param {Headers} headers
  * @param {Cookie} cookie
@@ -184,5 +194,6 @@ module.exports = {
   getCookies,
   deleteCookie,
   getSetCookies,
-  setCookie
+  setCookie,
+  parseCookie
 }

package/lib/web/cookies/parse.js

@@ -4,6 +4,7 @@ const { maxNameValuePairSize, maxAttributeValueSize } = require('./constants')
 const { isCTLExcludingHtab } = require('./util')
 const { collectASequenceOfCodePointsFast } = require('../fetch/data-url')
 const assert = require('node:assert')
+const { unescape } = require('node:querystring')
 
 /**
  * @description Parses the field-value attributes of a set-cookie header string.
@@ -76,8 +77,12 @@ function parseSetCookie (header) {
 
   // 6. The cookie-name is the name string, and the cookie-value is the
   //    value string.
+  // https://datatracker.ietf.org/doc/html/rfc6265
+  // To maximize compatibility with user agents, servers that wish to
+  // store arbitrary data in a cookie-value SHOULD encode that data, for
+  // example, using Base64 [RFC4648].
   return {
-    name, value, ...parseUnparsedAttributes(unparsedAttributes)
+    name, value: unescape(value), ...parseUnparsedAttributes(unparsedAttributes)
   }
 }
 

package/lib/web/fetch/body.js

@@ -364,12 +364,8 @@ function bodyMixinMethods (instance, getInternalState) {
           switch (mimeType.essence) {
             case 'multipart/form-data': {
               // 1. ... [long step]
-              const parsed = multipartFormDataParser(value, mimeType)
-
               // 2. If that fails for some reason, then throw a TypeError.
-              if (parsed === 'failure') {
-                throw new TypeError('Failed to parse body as FormData.')
-              }
+              const parsed = multipartFormDataParser(value, mimeType)
 
               // 3. Return a new FormData object, appending each entry,
               //    resulting from the parsing operation, to its entry list.