undici 7.0.0-alpha.6 → 7.0.0-alpha.7

package/docs/docs/api/CacheStore.md

@@ -13,9 +13,21 @@ The `MemoryCacheStore` stores the responses in-memory.
 
 **Options**
 
-- `maxEntries` - The maximum amount of responses to store. Default `Infinity`.
+- `maxCount` - The maximum amount of responses to store. Default `Infinity`.
 - `maxEntrySize` - The maximum size in bytes that a response's body can be. If a response's body is greater than or equal to this, the response will not be cached.
 
+### `SqliteCacheStore`
+
+The `SqliteCacheStore` stores the responses in a SQLite database.
+Under the hood, it uses Node.js' [`node:sqlite`](https://nodejs.org/api/sqlite.html) api.
+The `SqliteCacheStore` is only exposed if the `node:sqlite` api is present.
+
+**Options**
+
+- `location` - The location of the SQLite database to use. Default `:memory:`.
+- `maxCount` - The maximum number of entries to store in the database. Default `Infinity`.
+- `maxEntrySize` - The maximum size in bytes that a resposne's body can be. If a response's body is greater than or equal to this, the response will not be cached. Default `Infinity`.
+
 ## Defining a Custom Cache Store
 
 The store must implement the following functions:

package/docs/docs/api/Dispatcher.md

@@ -205,14 +205,12 @@ Returns: `Boolean` - `false` if dispatcher is busy and further dispatch calls wo
 
 #### Parameter: `DispatchHandler`
 
-* **onConnect** `(abort: () => void, context: object) => void` - Invoked before request is dispatched on socket. May be invoked multiple times when a request is retried when the request at the head of the pipeline fails.
-* **onError** `(error: Error) => void` - Invoked when an error has occurred. May not throw.
-* **onUpgrade** `(statusCode: number, headers: Buffer[], socket: Duplex) => void` (optional) - Invoked when request is upgraded. Required if `DispatchOptions.upgrade` is defined or `DispatchOptions.method === 'CONNECT'`.
-* **onResponseStarted** `() => void` (optional) - Invoked when response is received, before headers have been read.
-* **onHeaders** `(statusCode: number, headers: Buffer[], resume: () => void, statusText: string) => boolean` - Invoked when statusCode and headers have been received. May be invoked multiple times due to 1xx informational headers. Not required for `upgrade` requests.
-* **onData** `(chunk: Buffer) => boolean` - Invoked when response payload data is received. Not required for `upgrade` requests.
-* **onComplete** `(trailers: Buffer[]) => void` - Invoked when response payload and trailers have been received and the request has completed. Not required for `upgrade` requests.
-* **onBodySent** `(chunk: string | Buffer | Uint8Array) => void` - Invoked when a body chunk is sent to the server. Not required. For a stream or iterable body this will be invoked for every chunk. For other body types, it will be invoked once after the body is sent.
+* **onRequestStart** `(controller: DispatchController, context: object) => void` - Invoked before request is dispatched on socket. May be invoked multiple times when a request is retried when the request at the head of the pipeline fails.
+* **onRequestUpgrade** `(controller: DispatchController, statusCode: number, headers: Record<string, string | string[]>, socket: Duplex) => void` (optional) - Invoked when request is upgraded. Required if `DispatchOptions.upgrade` is defined or `DispatchOptions.method === 'CONNECT'`.
+* **onResponseStart** `(controller: DispatchController, statusCode: number, statusMessage?: string, headers: Record<string, string | string []>) => void` - Invoked when statusCode and headers have been received. May be invoked multiple times due to 1xx informational headers. Not required for `upgrade` requests.
+* **onResponseData** `(controller: DispatchController, chunk: Buffer) => void` - Invoked when response payload data is received. Not required for `upgrade` requests.
+* **onResponseEnd** `(controller: DispatchController, trailers: Record<string, string | string[]>) => void` - Invoked when response payload and trailers have been received and the request has completed. Not required for `upgrade` requests.
+* **onResponseError** `(error: Error) => void` - Invoked when an error has occurred. May not throw.
 
 #### Example 1 - Dispatch GET request
 

package/docs/docs/api/RedirectHandler.md

@@ -16,7 +16,7 @@ Returns: `RedirectHandler`
 
 ### Parameters
 
-- **dispatch** `(options: Dispatch.DispatchOptions, handlers: Dispatch.DispatchHandlers) => Promise<Dispatch.DispatchResponse>` (required) - Dispatch function to be called after every redirection.
+- **dispatch** `(options: Dispatch.DispatchOptions, handlers: Dispatch.DispatchHandler) => Promise<Dispatch.DispatchResponse>` (required) - Dispatch function to be called after every redirection.
 - **maxRedirections** `number` (required) - Maximum number of redirections allowed.
 - **opts** `object` (required) - Options for handling redirection.
 - **handler** `object` (required) - Handlers for different stages of the request lifecycle.

package/docs/docs/api/RetryHandler.md

@@ -43,8 +43,8 @@ It represents the retry state for a given request.
 
 ### Parameter `RetryHandlers`
 
-- **dispatch** `(options: Dispatch.DispatchOptions, handlers: Dispatch.DispatchHandlers) => Promise<Dispatch.DispatchResponse>` (required) - Dispatch function to be called after every retry.
-- **handler** Extends [`Dispatch.DispatchHandlers`](/docs/docs/api/Dispatcher.md#dispatcherdispatchoptions-handler) (required) - Handler function to be called after the request is successful or the retries are exhausted.
+- **dispatch** `(options: Dispatch.DispatchOptions, handlers: Dispatch.DispatchHandler) => Promise<Dispatch.DispatchResponse>` (required) - Dispatch function to be called after every retry.
+- **handler** Extends [`Dispatch.DispatchHandler`](/docs/docs/api/Dispatcher.md#dispatcherdispatchoptions-handler) (required) - Handler function to be called after the request is successful or the retries are exhausted.
 
 >__Note__: The `RetryHandler` does not retry over stateful bodies (e.g. streams, AsyncIterable) as those, once consumed, are left in a state that cannot be reutilized. For these situations the `RetryHandler` will identify
 >the body as stateful and will not retry the request rejecting with the error `UND_ERR_REQ_RETRY`.

package/index.js

@@ -48,6 +48,15 @@ module.exports.cacheStores = {
   MemoryCacheStore: require('./lib/cache/memory-cache-store')
 }
 
+try {
+  const SqliteCacheStore = require('./lib/cache/sqlite-cache-store')
+  module.exports.cacheStores.SqliteCacheStore = SqliteCacheStore
+} catch (err) {
+  if (err.code !== 'ERR_UNKNOWN_BUILTIN_MODULE') {
+    throw err
+  }
+}
+
 module.exports.buildConnector = buildConnector
 module.exports.errors = errors
 module.exports.util = {

package/lib/cache/memory-cache-store.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const { Writable } = require('node:stream')
+const { assertCacheKey, assertCacheValue } = require('../util/cache.js')
 
 /**
  * @typedef {import('../../types/cache-interceptor.d.ts').default.CacheKey} CacheKey
@@ -70,9 +71,7 @@ class MemoryCacheStore {
    * @returns {import('../../types/cache-interceptor.d.ts').default.GetResult | undefined}
    */
   get (key) {
-    if (typeof key !== 'object') {
-      throw new TypeError(`expected key to be object, got ${typeof key}`)
-    }
+    assertCacheKey(key)
 
     const topLevelKey = `${key.origin}:${key.path}`
 
@@ -88,7 +87,7 @@ class MemoryCacheStore {
       : {
           statusMessage: entry.statusMessage,
           statusCode: entry.statusCode,
-          rawHeaders: entry.rawHeaders,
+          headers: entry.headers,
           body: entry.body,
           etag: entry.etag,
           cachedAt: entry.cachedAt,
@@ -103,12 +102,8 @@ class MemoryCacheStore {
    * @returns {Writable | undefined}
    */
   createWriteStream (key, val) {
-    if (typeof key !== 'object') {
-      throw new TypeError(`expected key to be object, got ${typeof key}`)
-    }
-    if (typeof val !== 'object') {
-      throw new TypeError(`expected value to be object, got ${typeof val}`)
-    }
+    assertCacheKey(key)
+    assertCacheValue(val)
 
     const topLevelKey = `${key.origin}:${key.path}`
 

package/lib/cache/sqlite-cache-store.js

@@ -0,0 +1,457 @@
+'use strict'
+
+const { DatabaseSync } = require('node:sqlite')
+const { Writable } = require('stream')
+const { assertCacheKey, assertCacheValue } = require('../util/cache.js')
+
+const VERSION = 2
+
+/**
+ * @typedef {import('../../types/cache-interceptor.d.ts').default.CacheStore} CacheStore
+ * @implements {CacheStore}
+ *
+ * @typedef {{
+ *  id: Readonly<number>
+ *  headers?: Record<string, string | string[]>
+ *  vary?: string | object
+ *  body: string
+ * } & import('../../types/cache-interceptor.d.ts').default.CacheValue} SqliteStoreValue
+ */
+class SqliteCacheStore {
+  #maxEntrySize = Infinity
+  #maxCount = Infinity
+
+  /**
+   * @type {import('node:sqlite').DatabaseSync}
+   */
+  #db
+
+  /**
+   * @type {import('node:sqlite').StatementSync}
+   */
+  #getValuesQuery
+
+  /**
+   * @type {import('node:sqlite').StatementSync}
+   */
+  #updateValueQuery
+
+  /**
+   * @type {import('node:sqlite').StatementSync}
+   */
+  #insertValueQuery
+
+  /**
+   * @type {import('node:sqlite').StatementSync}
+   */
+  #deleteExpiredValuesQuery
+
+  /**
+   * @type {import('node:sqlite').StatementSync}
+   */
+  #deleteByUrlQuery
+
+  /**
+   * @type {import('node:sqlite').StatementSync}
+   */
+  #countEntriesQuery
+
+  /**
+   * @type {import('node:sqlite').StatementSync}
+   */
+  #deleteOldValuesQuery
+
+  /**
+   * @param {import('../../types/cache-interceptor.d.ts').default.SqliteCacheStoreOpts | undefined} opts
+   */
+  constructor (opts) {
+    if (opts) {
+      if (typeof opts !== 'object') {
+        throw new TypeError('SqliteCacheStore options must be an object')
+      }
+
+      if (opts.maxEntrySize !== undefined) {
+        if (
+          typeof opts.maxEntrySize !== 'number' ||
+          !Number.isInteger(opts.maxEntrySize) ||
+          opts.maxEntrySize < 0
+        ) {
+          throw new TypeError('SqliteCacheStore options.maxEntrySize must be a non-negative integer')
+        }
+        this.#maxEntrySize = opts.maxEntrySize
+      }
+
+      if (opts.maxCount !== undefined) {
+        if (
+          typeof opts.maxCount !== 'number' ||
+          !Number.isInteger(opts.maxCount) ||
+          opts.maxCount < 0
+        ) {
+          throw new TypeError('SqliteCacheStore options.maxCount must be a non-negative integer')
+        }
+        this.#maxCount = opts.maxCount
+      }
+    }
+
+    this.#db = new DatabaseSync(opts?.location ?? ':memory:')
+
+    this.#db.exec(`
+      CREATE TABLE IF NOT EXISTS cacheInterceptorV${VERSION} (
+        -- Data specific to us
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        url TEXT NOT NULL,
+        method TEXT NOT NULL,
+
+        -- Data returned to the interceptor
+        body TEXT NULL,
+        deleteAt INTEGER NOT NULL,
+        statusCode INTEGER NOT NULL,
+        statusMessage TEXT NOT NULL,
+        headers TEXT NULL,
+        etag TEXT NULL,
+        vary TEXT NULL,
+        cachedAt INTEGER NOT NULL,
+        staleAt INTEGER NOT NULL
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_cacheInterceptorV${VERSION}_url ON cacheInterceptorV${VERSION}(url);
+      CREATE INDEX IF NOT EXISTS idx_cacheInterceptorV${VERSION}_method ON cacheInterceptorV${VERSION}(method);
+      CREATE INDEX IF NOT EXISTS idx_cacheInterceptorV${VERSION}_deleteAt ON cacheInterceptorV${VERSION}(deleteAt);
+    `)
+
+    this.#getValuesQuery = this.#db.prepare(`
+      SELECT
+        id,
+        body,
+        deleteAt,
+        statusCode,
+        statusMessage,
+        headers,
+        etag,
+        vary,
+        cachedAt,
+        staleAt
+      FROM cacheInterceptorV${VERSION}
+      WHERE
+        url = ?
+        AND method = ?
+      ORDER BY
+        deleteAt ASC
+    `)
+
+    this.#updateValueQuery = this.#db.prepare(`
+      UPDATE cacheInterceptorV${VERSION} SET
+        body = ?,
+        deleteAt = ?,
+        statusCode = ?,
+        statusMessage = ?,
+        headers = ?,
+        etag = ?,
+        cachedAt = ?,
+        staleAt = ?,
+        deleteAt = ?
+      WHERE
+        id = ?
+    `)
+
+    this.#insertValueQuery = this.#db.prepare(`
+      INSERT INTO cacheInterceptorV${VERSION} (
+        url,
+        method,
+        body,
+        deleteAt,
+        statusCode,
+        statusMessage,
+        headers,
+        etag,
+        vary,
+        cachedAt,
+        staleAt,
+        deleteAt
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `)
+
+    this.#deleteByUrlQuery = this.#db.prepare(
+      `DELETE FROM cacheInterceptorV${VERSION} WHERE url = ?`
+    )
+
+    this.#countEntriesQuery = this.#db.prepare(
+      `SELECT COUNT(*) AS total FROM cacheInterceptorV${VERSION}`
+    )
+
+    this.#deleteExpiredValuesQuery = this.#db.prepare(
+      `DELETE FROM cacheInterceptorV${VERSION} WHERE deleteAt <= ?`
+    )
+
+    this.#deleteOldValuesQuery = this.#maxCount === Infinity
+      ? null
+      : this.#db.prepare(`
+        DELETE FROM cacheInterceptorV${VERSION}
+        WHERE id IN (
+          SELECT
+            id
+          FROM cacheInterceptorV${VERSION}
+          ORDER BY cachedAt DESC
+          LIMIT ?
+        )
+      `)
+  }
+
+  close () {
+    this.#db.close()
+  }
+
+  /**
+   * @param {import('../../types/cache-interceptor.d.ts').default.CacheKey} key
+   * @returns {import('../../types/cache-interceptor.d.ts').default.GetResult | undefined}
+   */
+  get (key) {
+    assertCacheKey(key)
+
+    const value = this.#findValue(key)
+
+    if (!value) {
+      return undefined
+    }
+
+    /**
+     * @type {import('../../types/cache-interceptor.d.ts').default.GetResult}
+     */
+    const result = {
+      body: value.body ? parseBufferArray(JSON.parse(value.body)) : null,
+      statusCode: value.statusCode,
+      statusMessage: value.statusMessage,
+      headers: value.headers ? JSON.parse(value.headers) : undefined,
+      etag: value.etag ? value.etag : undefined,
+      cachedAt: value.cachedAt,
+      staleAt: value.staleAt,
+      deleteAt: value.deleteAt
+    }
+
+    return result
+  }
+
+  /**
+   * @param {import('../../types/cache-interceptor.d.ts').default.CacheKey} key
+   * @param {import('../../types/cache-interceptor.d.ts').default.CacheValue} value
+   * @returns {Writable | undefined}
+   */
+  createWriteStream (key, value) {
+    assertCacheKey(key)
+    assertCacheValue(value)
+
+    const url = this.#makeValueUrl(key)
+    let size = 0
+    /**
+     * @type {Buffer[] | null}
+     */
+    const body = []
+    const store = this
+
+    return new Writable({
+      write (chunk, encoding, callback) {
+        if (typeof chunk === 'string') {
+          chunk = Buffer.from(chunk, encoding)
+        }
+
+        size += chunk.byteLength
+
+        if (size < store.#maxEntrySize) {
+          body.push(chunk)
+        } else {
+          this.destroy()
+        }
+
+        callback()
+      },
+      final (callback) {
+        const existingValue = store.#findValue(key, true)
+        if (existingValue) {
+          // Updating an existing response, let's overwrite it
+          store.#updateValueQuery.run(
+            JSON.stringify(stringifyBufferArray(body)),
+            value.deleteAt,
+            value.statusCode,
+            value.statusMessage,
+            value.headers ? JSON.stringify(value.headers) : null,
+            value.etag,
+            value.cachedAt,
+            value.staleAt,
+            value.deleteAt,
+            existingValue.id
+          )
+        } else {
+          store.#prune()
+          // New response, let's insert it
+          store.#insertValueQuery.run(
+            url,
+            key.method,
+            JSON.stringify(stringifyBufferArray(body)),
+            value.deleteAt,
+            value.statusCode,
+            value.statusMessage,
+            value.headers ? JSON.stringify(value.headers) : null,
+            value.etag ? value.etag : null,
+            value.vary ? JSON.stringify(value.vary) : null,
+            value.cachedAt,
+            value.staleAt,
+            value.deleteAt
+          )
+        }
+
+        callback()
+      }
+    })
+  }
+
+  /**
+   * @param {import('../../types/cache-interceptor.d.ts').default.CacheKey} key
+   */
+  delete (key) {
+    if (typeof key !== 'object') {
+      throw new TypeError(`expected key to be object, got ${typeof key}`)
+    }
+
+    this.#deleteByUrlQuery.run(this.#makeValueUrl(key))
+  }
+
+  #prune () {
+    if (this.#size <= this.#maxCount) {
+      return 0
+    }
+
+    {
+      const removed = this.#deleteExpiredValuesQuery.run(Date.now()).changes
+      if (removed > 0) {
+        return removed
+      }
+    }
+
+    {
+      const removed = this.#deleteOldValuesQuery.run(Math.max(Math.floor(this.#maxCount * 0.1), 1)).changes
+      if (removed > 0) {
+        return removed
+      }
+    }
+
+    return 0
+  }
+
+  /**
+   * Counts the number of rows in the cache
+   * @returns {Number}
+   */
+  get #size () {
+    const { total } = this.#countEntriesQuery.get()
+    return total
+  }
+
+  /**
+   * @param {import('../../types/cache-interceptor.d.ts').default.CacheKey} key
+   * @returns {string}
+   */
+  #makeValueUrl (key) {
+    return `${key.origin}/${key.path}`
+  }
+
+  /**
+   * @param {import('../../types/cache-interceptor.d.ts').default.CacheKey} key
+   * @param {boolean} [canBeExpired=false]
+   * @returns {(SqliteStoreValue & { vary?: Record<string, string[]> }) | undefined}
+   */
+  #findValue (key, canBeExpired = false) {
+    const url = this.#makeValueUrl(key)
+    const { headers, method } = key
+
+    /**
+     * @type {SqliteStoreValue[]}
+     */
+    const values = this.#getValuesQuery.all(url, method)
+
+    if (values.length === 0) {
+      return undefined
+    }
+
+    const now = Date.now()
+    for (const value of values) {
+      if (now >= value.deleteAt && !canBeExpired) {
+        return undefined
+      }
+
+      let matches = true
+
+      if (value.vary) {
+        if (!headers) {
+          return undefined
+        }
+
+        const vary = JSON.parse(value.vary)
+
+        for (const header in vary) {
+          if (headerValueEquals(headers[header], vary[header])) {
+            matches = false
+            break
+          }
+        }
+      }
+
+      if (matches) {
+        return value
+      }
+    }
+
+    return undefined
+  }
+}
+
+/**
+ * @param {string|string[]|null|undefined} lhs
+ * @param {string|string[]|null|undefined} rhs
+ * @returns {boolean}
+ */
+function headerValueEquals (lhs, rhs) {
+  if (Array.isArray(lhs) && Array.isArray(rhs)) {
+    if (lhs.length !== rhs.length) {
+      return false
+    }
+
+    for (let i = 0; i < lhs.length; i++) {
+      if (rhs.includes(lhs[i])) {
+        return false
+      }
+    }
+
+    return true
+  }
+
+  return lhs === rhs
+}
+
+/**
+ * @param {Buffer[]} buffers
+ * @returns {string[]}
+ */
+function stringifyBufferArray (buffers) {
+  const output = new Array(buffers.length)
+  for (let i = 0; i < buffers.length; i++) {
+    output[i] = buffers[i].toString()
+  }
+
+  return output
+}
+
+/**
+ * @param {string[]} strings
+ * @returns {Buffer[]}
+ */
+function parseBufferArray (strings) {
+  const output = new Array(strings.length)
+
+  for (let i = 0; i < strings.length; i++) {
+    output[i] = Buffer.from(strings[i])
+  }
+
+  return output
+}
+
+module.exports = SqliteCacheStore

package/lib/core/util.js

@@ -511,6 +511,11 @@ function assertRequestHandler (handler, method, upgrade) {
     throw new InvalidArgumentError('handler must be an object')
   }
 
+  if (typeof handler.onRequestStart === 'function') {
+    // TODO (fix): More checks...
+    return
+  }
+
   if (typeof handler.onConnect !== 'function') {
     throw new InvalidArgumentError('invalid onConnect method')
   }

package/lib/dispatcher/client-h1.js

@@ -769,8 +769,19 @@ async function connectH1 (client, socket) {
   client[kSocket] = socket
 
   if (!llhttpInstance) {
+    const noop = () => {}
+    socket.on('error', noop)
     llhttpInstance = await llhttpPromise
     llhttpPromise = null
+    socket.off('error', noop)
+  }
+
+  if (socket.errored) {
+    throw socket.errored
+  }
+
+  if (socket.destroyed) {
+    throw new SocketError('destroyed')
   }
 
   socket[kNoRef] = false

package/lib/dispatcher/client-h2.js

@@ -32,6 +32,8 @@ const {
 
 const kOpenStreams = Symbol('open streams')
 
+let extractBody
+
 // Experimental
 let h2ExperimentalWarned = false
 
@@ -201,11 +203,12 @@ function onHttp2SessionGoAway (errorCode) {
   util.destroy(this[kSocket], err)
 
   // Fail head of pipeline.
-  const request = client[kQueue][client[kRunningIdx]]
-  client[kQueue][client[kRunningIdx]++] = null
-  util.errorRequest(client, request, err)
-
-  client[kPendingIdx] = client[kRunningIdx]
+  if (client[kRunningIdx] < client[kQueue].length) {
+    const request = client[kQueue][client[kRunningIdx]]
+    client[kQueue][client[kRunningIdx]++] = null
+    util.errorRequest(client, request, err)
+    client[kPendingIdx] = client[kRunningIdx]
+  }
 
   assert(client[kRunning] === 0)
 
@@ -279,7 +282,8 @@ function shouldSendContentLength (method) {
 
 function writeH2 (client, request) {
   const session = client[kHTTP2Session]
-  const { body, method, path, host, upgrade, expectContinue, signal, headers: reqHeaders } = request
+  const { method, path, host, upgrade, expectContinue, signal, headers: reqHeaders } = request
+  let { body } = request
 
   if (upgrade) {
     util.errorRequest(client, request, new Error('Upgrade not supported for H2'))
@@ -407,6 +411,16 @@ function writeH2 (client, request) {
 
   let contentLength = util.bodyLength(body)
 
+  if (util.isFormDataLike(body)) {
+    extractBody ??= require('../web/fetch/body.js').extractBody
+
+    const [bodyStream, contentType] = extractBody(body)
+    headers['content-type'] = contentType
+
+    body = bodyStream.stream
+    contentLength = bodyStream.length
+  }
+
   if (contentLength == null) {
     contentLength = request.contentLength
   }

package/lib/dispatcher/dispatcher-base.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const Dispatcher = require('./dispatcher')
+const UnwrapHandler = require('../handler/unwrap-handler')
 const {
   ClientDestroyedError,
   ClientClosedError,
@@ -142,7 +143,7 @@ class DispatcherBase extends Dispatcher {
         throw new ClientClosedError()
       }
 
-      return this[kDispatch](opts, handler)
+      return this[kDispatch](opts, UnwrapHandler.unwrap(handler))
     } catch (err) {
       if (typeof handler.onError !== 'function') {
         throw new InvalidArgumentError('invalid onError method')