rekwest 2.4.0 → 3.0.0

package/README.md

@@ -8,12 +8,12 @@ and [http2.request](https://nodejs.org/api/http2.html#http2_clienthttp2session_r
 
 * Fetch-alike
 * Cool-beans config options (with defaults)
-* Automatic HTTP2 support (ALPN negotiation)
+* Automatic HTTP/2 support (ALPN negotiation)
 * Automatic or opt-in body parse (with non-UTF-8 charset decoding)
 * Automatic and simplistic `Cookies` treatment (with built-in jar)
 * Automatic decompression (with opt-in body compression)
 * Built-in streamable `File` & `FormData` interfaces
-* Support redirects with fine-grained tune-ups
+* Support redirects & retries with fine-grained tune-ups
 * Support all legit request body types (include blobs & streams)
 * Support both CJS and ESM module systems
 * Fully promise-able and pipe-able
@@ -120,54 +120,63 @@ console.log(res.body);
   extra [http2.ClientSessionOptions](https://nodejs.org/api/http2.html#http2_http2_connect_authority_options_listener)
   & [http2.ClientSessionRequestOptions](https://nodejs.org/api/http2.html#http2_clienthttp2session_request_headers_options)
   and [tls.ConnectionOptions](https://nodejs.org/api/tls.html#tls_tls_connect_options_callback)
-  for the HTTP2 attunes
+  for HTTP/2 attunes
   * `body` **{string | Array | AsyncIterator | Blob | Buffer | File | FromData | Iterator | Object | Readable |
-    Uint8Array | URLSearchParams}** Body to send with the request
-  * `cookies` **{boolean | Array[[key, value]] | Cookies | Object | URLSearchParams}** `Default: true` Cookies to add to
+    Uint8Array | URLSearchParams}** The body to send with the request
+  * `cookies` **{boolean | Array<[k, v]> | Cookies | Object | URLSearchParams}** `Default: true` The cookies to add to
     the request
-  * `digest` **{boolean}** `Default: true` Read response stream, or simply add a mixin
-  * `follow` **{number}** `Default: 20` Number of redirects to follow
-  * `h2` **{boolean}** `Default: false` Forces use of the HTTP2 protocol
-  * `headers` **{Object}** Headers to add to the request
-  * `parse` **{boolean}** `Default: true` Parse response body, or simply return a buffer
-  * `redirect` **{error | follow | manual}** `Default: 'follow'` Controls redirect flow
-  * `thenable` **{boolean}** `Default: false` Controls promise resolutions
+  * `digest` **{boolean}** `Default: true` Controls whether to read the response stream or just add a mixin
+  * `follow` **{number}** `Default: 20` The number of redirects to follow
+  * `h2` **{boolean}** `Default: false` Forces the use of HTTP/2 protocol
+  * `headers` **{Object}** The headers to add to the request
+  * `maxRetryAfter` **{number}** The upper limit of `retry-after` header. If unset, it will use `timeout` value
+  * `parse` **{boolean}** `Default: true` Controls whether to parse response body or simply return a buffer
+  * `redirect` **{error | follow | manual}** `Default: follow` Controls the redirect flows
+  * `retry` **{Object}** Represents the retry options
+    * `attempts` **{number}** `Default: 0` The number of retry attempts
+    * `backoffStrategy` **{string}** `Default: interval * Math.log(Math.random() * (Math.E * Math.E - Math.E) + Math.E)`
+      The backoff strategy algorithm that increases logarithmically. To fixate set value to `interval * 1`
+    * `interval` **{number}** `Default: 1e3` The initial retry interval
+    * `retryAfter` **{boolean}** `Default: true` Controls `retry-after` header receptiveness
+    * `statusCodes` **{number[]}** `Default: [429, 503]` The list of status codes to retry on
+  * `thenable` **{boolean}** `Default: false` Controls the promise resolutions
+  * `timeout` **{number}** `Default: 3e5` The number of milliseconds a request can take before termination
 * **Returns:** Promise that resolves to
   extended [http.IncomingMessage](https://nodejs.org/api/http.html#http_class_http_incomingmessage)
   or [http2.ClientHttp2Stream](https://nodejs.org/api/http2.html#http2_class_clienthttp2stream) which is respectively
   readable and duplex streams
   * if `degist: true` & `parse: true`
-    * `body` **{string | Array | Buffer | Object}** Body based on its content type
+    * `body` **{string | Array | Buffer | Object}** The body based on its content type
   * if `degist: false`
     * `arrayBuffer` **{AsyncFunction}** Reads the response and returns **ArrayBuffer**
     * `blob` **{AsyncFunction}** Reads the response and returns **Blob**
     * `body` **{AsyncFunction}** Reads the response and returns **Buffer** if `parse: false`
     * `json` **{AsyncFunction}** Reads the response and returns **Object**
     * `text` **{AsyncFunction}** Reads the response and returns **String**
-  * `bodyUsed` **{boolean}** Whether the response were read or not
-  * `cookies` **{undefined | Cookies}** Cookies sent and received with the response
-  * `headers` **{Object}** Headers received with the response
+  * `bodyUsed` **{boolean}** Indicates whether the response were read or not
+  * `cookies` **{undefined | Cookies}** The cookies sent and received with the response
+  * `headers` **{Object}** The headers received with the response
   * `httpVersion` **{string}** Indicates protocol version negotiated with the server
   * `ok` **{boolean}** Indicates if the response was successful (statusCode: **200-299**)
   * `redirected` **{boolean}** Indicates if the response is the result of a redirect
   * `statusCode` **{number}** Indicates the status code of the response
-  * `trailers` **{undefined | Object}** Trailer headers received with the response
+  * `trailers` **{undefined | Object}** The trailer headers received with the response
 
 ---
 
 #### `rekwest.defaults`
 
-Object to fill with default [options](#rekwesturl-options)
+The object to fulfill with default [options](#rekwesturl-options)
 
 ---
 
 #### `rekwest.stream(url[, options])`
 
-Method with limited functionality to use with streams and pipes
+The method with limited functionality to use with streams and/or pipes
 
 * No automata
 * No redirects
-* Pass `h2: true` in options to use the HTTP2 protocol
+* Pass `h2: true` in options to use HTTP/2 protocol
   * Or use `ackn({ url: URL })` method in advance to probe the available protocols
 
 ---

package/dist/errors.js

@@ -1,7 +1,7 @@
 "use strict";
 
 exports.__esModule = true;
-exports.RequestError = void 0;
+exports.TimeoutError = exports.RequestError = void 0;
 
 class RequestError extends Error {
   get [Symbol.toStringTag]() {
@@ -12,11 +12,15 @@ class RequestError extends Error {
     return this[Symbol.toStringTag];
   }
 
-  constructor(message) {
-    super(message);
+  constructor(...args) {
+    super(...args);
     Error.captureStackTrace(this, this.constructor);
   }
 
 }
 
-exports.RequestError = RequestError;
+exports.RequestError = RequestError;
+
+class TimeoutError extends RequestError {}
+
+exports.TimeoutError = TimeoutError;

package/dist/helpers.js

@@ -1,7 +1,7 @@
 "use strict";
 
 exports.__esModule = true;
-exports.redirects = exports.premix = exports.preflight = exports.merge = exports.dispatch = exports.decompress = exports.compress = void 0;
+exports.redirects = exports.preflight = exports.mixin = exports.merge = exports.dispatch = exports.decompress = exports.compress = exports.affix = exports.admix = void 0;
 exports.tap = tap;
 exports.transform = void 0;
 
@@ -17,6 +17,8 @@ var _zlib = _interopRequireDefault(require("zlib"));
 
 var _cookies = require("./cookies.js");
 
+var _errors = require("./errors.js");
+
 var _file = require("./file.js");
 
 var _formdata = require("./formdata.js");
@@ -36,6 +38,7 @@ const {
   HTTP2_HEADER_METHOD,
   HTTP2_HEADER_PATH,
   HTTP2_HEADER_SCHEME,
+  HTTP2_HEADER_STATUS,
   HTTP2_METHOD_GET,
   HTTP2_METHOD_HEAD
 } = _http.default.constants;
@@ -46,6 +49,51 @@ const gunzip = (0, _util.promisify)(_zlib.default.gunzip);
 const deflate = (0, _util.promisify)(_zlib.default.deflate);
 const inflate = (0, _util.promisify)(_zlib.default.inflate);
 
+const admix = (res, headers, options) => {
+  const {
+    h2
+  } = options;
+
+  if (h2) {
+    Reflect.defineProperty(res, 'headers', {
+      enumerable: true,
+      value: headers
+    });
+    Reflect.defineProperty(res, 'httpVersion', {
+      enumerable: true,
+      value: `${h2 + 1}.0`
+    });
+    Reflect.defineProperty(res, 'statusCode', {
+      enumerable: true,
+      value: headers[HTTP2_HEADER_STATUS]
+    });
+  }
+
+  Reflect.defineProperty(res, 'ok', {
+    enumerable: true,
+    value: /^2\d{2}$/.test(res.statusCode)
+  });
+  Reflect.defineProperty(res, 'redirected', {
+    enumerable: true,
+    value: !!options.redirected
+  });
+};
+
+exports.admix = admix;
+
+const affix = (client, req, options) => {
+  req.once('end', () => client?.close());
+  req.once('timeout', () => req.destroy(new _errors.TimeoutError(`Timed out after ${options.timeout} ms.`)));
+  req.once('trailers', trailers => {
+    Reflect.defineProperty(req, 'trailers', {
+      enumerable: true,
+      value: trailers
+    });
+  });
+};
+
+exports.affix = affix;
+
 const compress = (buf, encoding, {
   async = false
 } = {}) => {
@@ -127,73 +175,7 @@ const merge = (target = {}, ...rest) => {
 
 exports.merge = merge;
 
-const preflight = options => {
-  const url = options.url = new URL(options.url);
-  const {
-    cookies,
-    h2 = false,
-    method = HTTP2_METHOD_GET,
-    headers,
-    redirected
-  } = options;
-
-  if (h2) {
-    options.endStream = [HTTP2_METHOD_GET, HTTP2_METHOD_HEAD].includes(method);
-  }
-
-  if (cookies !== false) {
-    let cookie = _cookies.Cookies.jar.get(url.origin);
-
-    if (cookies === Object(cookies) && !redirected) {
-      if (cookie) {
-        new _cookies.Cookies(cookies).forEach(function (val, key) {
-          this.set(key, val);
-        }, cookie);
-      } else {
-        cookie = new _cookies.Cookies(cookies);
-
-        _cookies.Cookies.jar.set(url.origin, cookie);
-      }
-    }
-
-    options.headers = { ...(cookie && {
-        [HTTP2_HEADER_COOKIE]: cookie
-      }),
-      ...headers
-    };
-  }
-
-  options.digest ??= true;
-  options.follow ??= 20;
-  options.h2 ??= h2;
-  options.headers = {
-    [HTTP2_HEADER_ACCEPT]: `${_mediatypes.APPLICATION_JSON}, ${_mediatypes.TEXT_PLAIN}, ${_mediatypes.WILDCARD}`,
-    [HTTP2_HEADER_ACCEPT_ENCODING]: 'br, deflate, gzip, identity',
-    ...Object.entries(options.headers ?? {}).reduce((acc, [key, val]) => (acc[key.toLowerCase()] = val, acc), {}),
-    ...(h2 && {
-      [HTTP2_HEADER_AUTHORITY]: url.host,
-      [HTTP2_HEADER_METHOD]: method,
-      [HTTP2_HEADER_PATH]: `${url.pathname}${url.search}`,
-      [HTTP2_HEADER_SCHEME]: url.protocol.replace(/\p{Punctuation}/gu, '')
-    })
-  };
-  options.method ??= method;
-  options.parse ??= true;
-  options.redirect ??= redirects.follow;
-
-  if (!Object.values(redirects).includes(options.redirect)) {
-    options.createConnection?.().destroy();
-    throw new TypeError(`Failed to read the 'redirect' property from 'options': The provided value '${options.redirect}' is not a valid enum value.`);
-  }
-
-  options.redirected ??= false;
-  options.thenable ??= false;
-  return options;
-};
-
-exports.preflight = preflight;
-
-const premix = (res, {
+const mixin = (res, {
   digest = false,
   parse = false
 } = {}) => {
@@ -281,7 +263,73 @@ const premix = (res, {
   });
 };
 
-exports.premix = premix;
+exports.mixin = mixin;
+
+const preflight = options => {
+  const url = options.url = new URL(options.url);
+  const {
+    cookies,
+    h2 = false,
+    method = HTTP2_METHOD_GET,
+    headers,
+    redirected
+  } = options;
+
+  if (h2) {
+    options.endStream = [HTTP2_METHOD_GET, HTTP2_METHOD_HEAD].includes(method);
+  }
+
+  if (cookies !== false) {
+    let cookie = _cookies.Cookies.jar.get(url.origin);
+
+    if (cookies === Object(cookies) && !redirected) {
+      if (cookie) {
+        new _cookies.Cookies(cookies).forEach(function (val, key) {
+          this.set(key, val);
+        }, cookie);
+      } else {
+        cookie = new _cookies.Cookies(cookies);
+
+        _cookies.Cookies.jar.set(url.origin, cookie);
+      }
+    }
+
+    options.headers = { ...(cookie && {
+        [HTTP2_HEADER_COOKIE]: cookie
+      }),
+      ...headers
+    };
+  }
+
+  options.digest ??= true;
+  options.follow ??= 20;
+  options.h2 ??= h2;
+  options.headers = {
+    [HTTP2_HEADER_ACCEPT]: `${_mediatypes.APPLICATION_JSON}, ${_mediatypes.TEXT_PLAIN}, ${_mediatypes.WILDCARD}`,
+    [HTTP2_HEADER_ACCEPT_ENCODING]: 'br, deflate, gzip, identity',
+    ...Object.entries(options.headers ?? {}).reduce((acc, [key, val]) => (acc[key.toLowerCase()] = val, acc), {}),
+    ...(h2 && {
+      [HTTP2_HEADER_AUTHORITY]: url.host,
+      [HTTP2_HEADER_METHOD]: method,
+      [HTTP2_HEADER_PATH]: `${url.pathname}${url.search}`,
+      [HTTP2_HEADER_SCHEME]: url.protocol.replace(/\p{Punctuation}/gu, '')
+    })
+  };
+  options.method ??= method;
+  options.parse ??= true;
+  options.redirect ??= redirects.follow;
+
+  if (!Object.values(redirects).includes(options.redirect)) {
+    options.createConnection?.().destroy();
+    throw new TypeError(`Failed to read the 'redirect' property from 'options': The provided value '${options.redirect}' is not a valid enum value.`);
+  }
+
+  options.redirected ??= false;
+  options.thenable ??= false;
+  return options;
+};
+
+exports.preflight = preflight;
 const redirects = {
   error: 'error',
   follow: 'follow',

package/dist/index.js

@@ -15,6 +15,8 @@ exports.constants = _http2.constants;
 
 var _https = _interopRequireDefault(require("https"));
 
+var _promises = require("timers/promises");
+
 var _ackn = require("./ackn.js");
 
 Object.keys(_ackn).forEach(function (key) {
@@ -77,29 +79,55 @@ const {
   HTTP2_HEADER_CONTENT_LENGTH,
   HTTP2_HEADER_CONTENT_TYPE,
   HTTP2_HEADER_LOCATION,
+  HTTP2_HEADER_RETRY_AFTER,
   HTTP2_HEADER_SET_COOKIE,
   HTTP2_HEADER_STATUS,
   HTTP2_METHOD_GET,
   HTTP2_METHOD_HEAD,
   HTTP_STATUS_BAD_REQUEST,
-  HTTP_STATUS_SEE_OTHER
+  HTTP_STATUS_MOVED_PERMANENTLY,
+  HTTP_STATUS_SEE_OTHER,
+  HTTP_STATUS_SERVICE_UNAVAILABLE,
+  HTTP_STATUS_TOO_MANY_REQUESTS
 } = _http2.default.constants;
+const maxRetryAfter = Symbol('maxRetryAfter');
+
+const maxRetryAfterError = (interval, options) => new _errors.RequestError(`Maximum '${HTTP2_HEADER_RETRY_AFTER}' limit exceeded: ${interval} ms.`, options);
+
+let defaults = {
+  follow: 20,
+
+  get maxRetryAfter() {
+    return this[maxRetryAfter] ?? this.timeout;
+  },
+
+  set maxRetryAfter(value) {
+    this[maxRetryAfter] = value;
+  },
+
+  method: HTTP2_METHOD_GET,
+  retry: {
+    attempts: 0,
+    backoffStrategy: 'interval * Math.log(Math.random() * (Math.E * Math.E - Math.E) + Math.E)',
+    interval: 1e3,
+    retryAfter: true,
+    statusCodes: [HTTP_STATUS_TOO_MANY_REQUESTS, HTTP_STATUS_SERVICE_UNAVAILABLE]
+  },
+  timeout: 3e5
+};
 
 async function rekwest(url, options = {}) {
   url = options.url = new URL(url);
 
   if (!options.redirected) {
-    options = (0, _helpers.merge)(rekwest.defaults, {
-      follow: 20,
-      method: HTTP2_METHOD_GET
-    }, options);
+    options = (0, _helpers.merge)(rekwest.defaults, options);
   }
 
   if (options.body && [HTTP2_METHOD_GET, HTTP2_METHOD_HEAD].includes(options.method)) {
     throw new TypeError(`Request with ${HTTP2_METHOD_GET}/${HTTP2_METHOD_HEAD} method cannot have body.`);
   }
 
-  if (!options.follow) {
+  if (options.follow === 0) {
     throw new _errors.RequestError(`Maximum redirect reached at: ${url.href}`);
   }
 
@@ -139,26 +167,22 @@ async function rekwest(url, options = {}) {
       req = request(url, options);
     }
 
-    req.on('response', res => {
+    (0, _helpers.affix)(client, req, options);
+    req.once('error', reject);
+    req.once('frameError', reject);
+    req.once('goaway', reject);
+    req.once('response', res => {
+      let headers;
+
       if (h2) {
-        const headers = res;
+        headers = res;
         res = req;
-        Reflect.defineProperty(res, 'headers', {
-          enumerable: true,
-          value: headers
-        });
-        Reflect.defineProperty(res, 'httpVersion', {
-          enumerable: true,
-          value: `${h2 + 1}.0`
-        });
-        Reflect.defineProperty(res, 'statusCode', {
-          enumerable: true,
-          value: headers[HTTP2_HEADER_STATUS]
-        });
       } else {
-        res.on('error', reject);
+        res.once('error', reject);
       }
 
+      (0, _helpers.admix)(res, headers, options);
+
       if (cookies !== false && res.headers[HTTP2_HEADER_SET_COOKIE]) {
         if (_cookies.Cookies.jar.has(url.origin)) {
           new _cookies.Cookies(res.headers[HTTP2_HEADER_SET_COOKIE]).forEach(function (val, key) {
@@ -195,37 +219,29 @@ async function rekwest(url, options = {}) {
           }
 
           Reflect.set(options, 'redirected', true);
+
+          if (res.statusCode === HTTP_STATUS_MOVED_PERMANENTLY && res.headers[HTTP2_HEADER_RETRY_AFTER]) {
+            let interval = res.headers[HTTP2_HEADER_RETRY_AFTER];
+            interval = Number(interval) * 1000 || new Date(interval) - Date.now();
+
+            if (interval > options.maxRetryAfter) {
+              res.emit('error', maxRetryAfterError(interval, {
+                cause: (0, _helpers.mixin)(res, options)
+              }));
+            }
+
+            return (0, _promises.setTimeout)(interval).then(() => rekwest(options.url, options).then(resolve, reject));
+          }
+
           return rekwest(options.url, options).then(resolve, reject);
         }
       }
 
-      Reflect.defineProperty(res, 'ok', {
-        enumerable: true,
-        value: /^2\d{2}$/.test(res.statusCode)
-      });
-      Reflect.defineProperty(res, 'redirected', {
-        enumerable: true,
-        value: options.redirected
-      });
-
       if (res.statusCode >= HTTP_STATUS_BAD_REQUEST) {
-        return reject((0, _helpers.premix)(res, options));
+        return reject((0, _helpers.mixin)(res, options));
       }
 
-      resolve((0, _helpers.premix)(res, options));
-    });
-    req.on('end', () => {
-      client?.close();
-    });
-    req.on('error', reject);
-    req.on('frameError', reject);
-    req.on('goaway', reject);
-    req.on('timeout', req.destroy);
-    req.on('trailers', trailers => {
-      Reflect.defineProperty(req, 'trailers', {
-        enumerable: true,
-        value: trailers
-      });
+      resolve((0, _helpers.mixin)(res, options));
     });
     (0, _helpers.dispatch)(req, { ...options,
       body
@@ -241,6 +257,29 @@ async function rekwest(url, options = {}) {
 
     return res;
   } catch (ex) {
+    if (options.retry?.attempts && options.retry?.statusCodes.includes(ex.statusCode)) {
+      let {
+        interval
+      } = options.retry;
+
+      if (options.retry.retryAfter && ex.headers[HTTP2_HEADER_RETRY_AFTER]) {
+        interval = ex.headers[HTTP2_HEADER_RETRY_AFTER];
+        interval = Number(interval) * 1000 || new Date(interval) - Date.now();
+
+        if (interval > options.maxRetryAfter) {
+          throw maxRetryAfterError(interval, {
+            cause: ex
+          });
+        }
+      } else {
+        interval = new Function('interval', `return Math.ceil(${options.retry.backoffStrategy});`)(interval);
+      }
+
+      options.retry.attempts--;
+      options.retry.interval = interval;
+      return (0, _promises.setTimeout)(interval).then(() => rekwest(url, options));
+    }
+
     if (digest && !redirected && ex.body) {
       ex.body = await ex.body();
     }
@@ -265,18 +304,8 @@ Reflect.defineProperty(rekwest, 'stream', {
       }, options),
       redirect: _helpers.redirects.manual
     });
-
-    if (options.h2) {
-      const client = _http2.default.connect(url.origin, options);
-
-      const req = client.request(options.headers, options);
-      req.on('end', () => {
-        client.close();
-      });
-      return req;
-    }
-
     const {
+      h2,
       url: {
         protocol
       }
@@ -284,11 +313,38 @@ Reflect.defineProperty(rekwest, 'stream', {
     const {
       request
     } = protocol === 'http:' ? _http.default : _https.default;
-    return request(options.url, options);
+    let client, req;
+
+    if (h2) {
+      client = _http2.default.connect(url.origin, options);
+      req = client.request(options.headers, options);
+    } else {
+      req = request(options.url, options);
+    }
+
+    (0, _helpers.affix)(client, req, options);
+    req.once('response', res => {
+      let headers;
+
+      if (h2) {
+        headers = res;
+        res = req;
+      }
+
+      (0, _helpers.admix)(res, headers, options);
+    });
+    return req;
   }
 });
 Reflect.defineProperty(rekwest, 'defaults', {
   enumerable: true,
-  value: Object.create(null),
-  writable: true
+
+  get() {
+    return defaults;
+  },
+
+  set(value) {
+    defaults = (0, _helpers.merge)(defaults, value);
+  }
+
 });

package/package.json

@@ -8,14 +8,14 @@
     "url": "https://github.com/bricss/rekwest/issues"
   },
   "devDependencies": {
-    "@babel/cli": "^7.16.8",
-    "@babel/core": "^7.16.12",
-    "@babel/eslint-parser": "^7.16.5",
+    "@babel/cli": "^7.17.6",
+    "@babel/core": "^7.17.5",
+    "@babel/eslint-parser": "^7.17.0",
     "@babel/preset-env": "^7.16.11",
     "c8": "^7.11.0",
-    "eslint": "^8.8.0",
-    "eslint-config-ultra-refined": "^2.4.0",
-    "mocha": "^9.2.0"
+    "eslint": "^8.10.0",
+    "eslint-config-ultra-refined": "^2.4.1",
+    "mocha": "^9.2.1"
   },
   "description": "The robust request library that humanity deserves 🌐",
   "engines": {
@@ -60,5 +60,5 @@
     "test:bail": "mocha --bail",
     "test:cover": "c8 --include=src --reporter=lcov --reporter=text npm test"
   },
-  "version": "2.4.0"
+  "version": "3.0.0"
 }