rekwest 2.3.6 → 3.0.1

package/src/helpers.mjs

@@ -10,6 +10,7 @@ import {
 } from 'util';
 import zlib from 'zlib';
 import { Cookies } from './cookies.mjs';
+import { TimeoutError } from './errors.mjs';
 import { File } from './file.mjs';
 import { FormData } from './formdata.mjs';
 import {
@@ -31,6 +32,7 @@ const {
   HTTP2_HEADER_METHOD,
   HTTP2_HEADER_PATH,
   HTTP2_HEADER_SCHEME,
+  HTTP2_HEADER_STATUS,
   HTTP2_METHOD_GET,
   HTTP2_METHOD_HEAD,
 } = http2.constants;
@@ -42,6 +44,48 @@ const gunzip = promisify(zlib.gunzip);
 const deflate = promisify(zlib.deflate);
 const inflate = promisify(zlib.inflate);
 
+export const admix = (res, headers, options) => {
+  const { h2 } = options;
+
+  if (h2) {
+    Reflect.defineProperty(res, 'headers', {
+      enumerable: true,
+      value: headers,
+    });
+
+    Reflect.defineProperty(res, 'httpVersion', {
+      enumerable: true,
+      value: `${ h2 + 1 }.0`,
+    });
+
+    Reflect.defineProperty(res, 'statusCode', {
+      enumerable: true,
+      value: headers[HTTP2_HEADER_STATUS],
+    });
+  }
+
+  Reflect.defineProperty(res, 'ok', {
+    enumerable: true,
+    value: /^2\d{2}$/.test(res.statusCode),
+  });
+
+  Reflect.defineProperty(res, 'redirected', {
+    enumerable: true,
+    value: !!options.redirected,
+  });
+};
+
+export const affix = (client, req, options) => {
+  req.once('end', () => client?.close());
+  req.once('timeout', () => req.destroy(new TimeoutError(`Timed out after ${ options.timeout } ms.`)));
+  req.once('trailers', (trailers) => {
+    Reflect.defineProperty(req, 'trailers', {
+      enumerable: true,
+      value: trailers,
+    });
+  });
+};
+
 export const compress = (buf, encoding, { async = false } = {}) => {
   encoding &&= encoding.match(/(?<encoding>\bbr\b|\bdeflate\b|\bgzip\b)/i)?.groups.encoding.toLowerCase();
   const compressor = {
@@ -118,63 +162,7 @@ export const merge = (target = {}, ...rest) => {
   return target;
 };
 
-export const preflight = (options) => {
-  const url = options.url = new URL(options.url);
-  const { cookies, h2 = false, method = HTTP2_METHOD_GET, headers, redirected } = options;
-
-  if (h2) {
-    options.endStream = [
-      HTTP2_METHOD_GET,
-      HTTP2_METHOD_HEAD,
-    ].includes(method);
-  }
-
-  if (cookies !== false) {
-    let cookie = Cookies.jar.get(url.origin);
-
-    if (cookies === Object(cookies) && !redirected) {
-      if (cookie) {
-        new Cookies(cookies).forEach(function (val, key) {
-          this.set(key, val);
-        }, cookie);
-      } else {
-        cookie = new Cookies(cookies);
-        Cookies.jar.set(url.origin, cookie);
-      }
-    }
-
-    options.headers = {
-      ...cookie && { [HTTP2_HEADER_COOKIE]: cookie },
-      ...headers,
-    };
-  }
-
-  options.digest ??= true;
-  options.follow ??= 20;
-  options.h2 ??= h2;
-  options.headers = {
-    [HTTP2_HEADER_ACCEPT]: `${ APPLICATION_JSON }, ${ TEXT_PLAIN }, ${ WILDCARD }`,
-    [HTTP2_HEADER_ACCEPT_ENCODING]: 'br, deflate, gzip, identity',
-    ...Object.entries(options.headers ?? {})
-             .reduce((acc, [key, val]) => (acc[key.toLowerCase()] = val, acc), {}),
-    ...h2 && {
-      [HTTP2_HEADER_AUTHORITY]: url.host,
-      [HTTP2_HEADER_METHOD]: method,
-      [HTTP2_HEADER_PATH]: `${ url.pathname }${ url.search }`,
-      [HTTP2_HEADER_SCHEME]: url.protocol.replace(/\p{Punctuation}/gu, ''),
-    },
-  };
-
-  options.method ??= method;
-  options.parse ??= true;
-  options.redirect ??= 'follow';
-  options.redirected ??= false;
-  options.thenable ??= false;
-
-  return options;
-};
-
-export const premix = (res, { digest = false, parse = false } = {}) => {
+export const mixin = (res, { digest = false, parse = false } = {}) => {
   if (!digest) {
     Object.defineProperties(res, {
       arrayBuffer: {
@@ -214,7 +202,7 @@ export const premix = (res, { digest = false, parse = false } = {}) => {
       enumerable: true,
       value: async function () {
         if (this.bodyUsed) {
-          throw new TypeError('Response stream already read');
+          throw new TypeError('Response stream already read.');
         }
 
         let spool = [];
@@ -262,6 +250,76 @@ export const premix = (res, { digest = false, parse = false } = {}) => {
   });
 };
 
+export const preflight = (options) => {
+  const url = options.url = new URL(options.url);
+  const { cookies, h2 = false, method = HTTP2_METHOD_GET, headers, redirected } = options;
+
+  if (h2) {
+    options.endStream = [
+      HTTP2_METHOD_GET,
+      HTTP2_METHOD_HEAD,
+    ].includes(method);
+  }
+
+  if (cookies !== false) {
+    let cookie = Cookies.jar.get(url.origin);
+
+    if (cookies === Object(cookies) && !redirected) {
+      if (cookie) {
+        new Cookies(cookies).forEach(function (val, key) {
+          this.set(key, val);
+        }, cookie);
+      } else {
+        cookie = new Cookies(cookies);
+        Cookies.jar.set(url.origin, cookie);
+      }
+    }
+
+    options.headers = {
+      ...cookie && { [HTTP2_HEADER_COOKIE]: cookie },
+      ...headers,
+    };
+  }
+
+  options.digest ??= true;
+  options.follow ??= 20;
+  options.h2 ??= h2;
+  options.headers = {
+    [HTTP2_HEADER_ACCEPT]: `${ APPLICATION_JSON }, ${ TEXT_PLAIN }, ${ WILDCARD }`,
+    [HTTP2_HEADER_ACCEPT_ENCODING]: 'br, deflate, gzip, identity',
+    ...Object.entries(options.headers ?? {})
+             .reduce((acc, [key, val]) => (acc[key.toLowerCase()] = val, acc), {}),
+    ...h2 && {
+      [HTTP2_HEADER_AUTHORITY]: url.host,
+      [HTTP2_HEADER_METHOD]: method,
+      [HTTP2_HEADER_PATH]: `${ url.pathname }${ url.search }`,
+      [HTTP2_HEADER_SCHEME]: url.protocol.replace(/\p{Punctuation}/gu, ''),
+    },
+  };
+
+  options.method ??= method;
+  options.parse ??= true;
+  options.redirect ??= redirects.follow;
+
+  if (!Object.values(redirects).includes(options.redirect)) {
+    options.createConnection?.().destroy();
+    throw new TypeError(`Failed to read the 'redirect' property from 'options': The provided value '${
+      options.redirect
+    }' is not a valid enum value.`);
+  }
+
+  options.redirected ??= false;
+  options.thenable ??= false;
+
+  return options;
+};
+
+export const redirects = {
+  error: 'error',
+  follow: 'follow',
+  manual: 'manual',
+};
+
 export async function* tap(value) {
   if (Reflect.has(value, Symbol.asyncIterator)) {
     yield* value;

package/src/index.mjs

@@ -1,14 +1,18 @@
 import http from 'http';
 import http2 from 'http2';
 import https from 'https';
+import { setTimeout as setTimeoutPromise } from 'timers/promises';
 import { ackn } from './ackn.mjs';
 import { Cookies } from './cookies.mjs';
 import { RequestError } from './errors.mjs';
 import {
+  admix,
+  affix,
   dispatch,
   merge,
+  mixin,
   preflight,
-  premix,
+  redirects,
   transform,
 } from './helpers.mjs';
 import { APPLICATION_OCTET_STREAM } from './mediatypes.mjs';
@@ -26,28 +30,58 @@ const {
   HTTP2_HEADER_CONTENT_LENGTH,
   HTTP2_HEADER_CONTENT_TYPE,
   HTTP2_HEADER_LOCATION,
+  HTTP2_HEADER_RETRY_AFTER,
   HTTP2_HEADER_SET_COOKIE,
-  HTTP2_HEADER_STATUS,
   HTTP2_METHOD_GET,
   HTTP2_METHOD_HEAD,
   HTTP_STATUS_BAD_REQUEST,
+  HTTP_STATUS_MOVED_PERMANENTLY,
   HTTP_STATUS_SEE_OTHER,
+  HTTP_STATUS_SERVICE_UNAVAILABLE,
+  HTTP_STATUS_TOO_MANY_REQUESTS,
 } = http2.constants;
 
+const maxRetryAfter = Symbol('maxRetryAfter');
+const maxRetryAfterError = (
+  interval,
+  options,
+) => new RequestError(`Maximum '${ HTTP2_HEADER_RETRY_AFTER }' limit exceeded: ${ interval } ms.`, options);
+let defaults = {
+  follow: 20,
+  get maxRetryAfter() {
+    return this[maxRetryAfter] ?? this.timeout;
+  },
+  set maxRetryAfter(value) {
+    this[maxRetryAfter] = value;
+  },
+  method: HTTP2_METHOD_GET,
+  retry: {
+    attempts: 0,
+    backoffStrategy: 'interval * Math.log(Math.random() * (Math.E * Math.E - Math.E) + Math.E)',
+    interval: 1e3,
+    retryAfter: true,
+    statusCodes: [
+      HTTP_STATUS_TOO_MANY_REQUESTS,
+      HTTP_STATUS_SERVICE_UNAVAILABLE,
+    ],
+  },
+  timeout: 3e5,
+};
+
 export default async function rekwest(url, options = {}) {
   url = options.url = new URL(url);
   if (!options.redirected) {
-    options = merge(rekwest.defaults, { follow: 20, method: HTTP2_METHOD_GET }, options);
+    options = merge(rekwest.defaults, options);
   }
 
   if (options.body && [
     HTTP2_METHOD_GET,
     HTTP2_METHOD_HEAD,
   ].includes(options.method)) {
-    throw new TypeError(`Request with ${ HTTP2_METHOD_GET }/${ HTTP2_METHOD_HEAD } method cannot have body`);
+    throw new TypeError(`Request with ${ HTTP2_METHOD_GET }/${ HTTP2_METHOD_HEAD } method cannot have body.`);
   }
 
-  if (!options.follow) {
+  if (options.follow === 0) {
     throw new RequestError(`Maximum redirect reached at: ${ url.href }`);
   }
 
@@ -80,30 +114,22 @@ export default async function rekwest(url, options = {}) {
       req = request(url, options);
     }
 
-    req.on('response', (res) => {
-      if (h2) {
-        const headers = res;
+    affix(client, req, options);
+    req.once('error', reject);
+    req.once('frameError', reject);
+    req.once('goaway', reject);
+    req.once('response', (res) => {
+      let headers;
 
+      if (h2) {
+        headers = res;
         res = req;
-
-        Reflect.defineProperty(res, 'headers', {
-          enumerable: true,
-          value: headers,
-        });
-
-        Reflect.defineProperty(res, 'httpVersion', {
-          enumerable: true,
-          value: `${ h2 + 1 }.0`,
-        });
-
-        Reflect.defineProperty(res, 'statusCode', {
-          enumerable: true,
-          value: headers[HTTP2_HEADER_STATUS],
-        });
       } else {
-        res.on('error', reject);
+        res.once('error', reject);
       }
 
+      admix(res, headers, options);
+
       if (cookies !== false && res.headers[HTTP2_HEADER_SET_COOKIE]) {
         if (Cookies.jar.has(url.origin)) {
           new Cookies(res.headers[HTTP2_HEADER_SET_COOKIE]).forEach(function (val, key) {
@@ -122,16 +148,16 @@ export default async function rekwest(url, options = {}) {
       });
 
       if (follow && /^3\d{2}$/.test(res.statusCode) && res.headers[HTTP2_HEADER_LOCATION]) {
-        if (redirect === 'error') {
-          res.emit('error', new RequestError(`Unexpected redirect, redirect mode is set to '${ redirect }'`));
+        if (redirect === redirects.error) {
+          res.emit('error', new RequestError(`Unexpected redirect, redirect mode is set to '${ redirect }'.`));
         }
 
-        if (redirect === 'follow') {
+        if (redirect === redirects.follow) {
           options.url = new URL(res.headers[HTTP2_HEADER_LOCATION], url).href;
 
           if (res.statusCode !== HTTP_STATUS_SEE_OTHER
             && body === Object(body) && body.pipe?.constructor === Function) {
-            res.emit('error', new RequestError(`Unable to ${ redirect } redirect with body as readable stream`));
+            res.emit('error', new RequestError(`Unable to ${ redirect } redirect with body as readable stream.`));
           }
 
           options.follow--;
@@ -144,39 +170,27 @@ export default async function rekwest(url, options = {}) {
 
           Reflect.set(options, 'redirected', true);
 
+          if (res.statusCode === HTTP_STATUS_MOVED_PERMANENTLY && res.headers[HTTP2_HEADER_RETRY_AFTER]) {
+            let interval = res.headers[HTTP2_HEADER_RETRY_AFTER];
+
+            interval = Number(interval) * 1000 || new Date(interval) - Date.now();
+
+            if (interval > options.maxRetryAfter) {
+              res.emit('error', maxRetryAfterError(interval, { cause: mixin(res, options) }));
+            }
+
+            return setTimeoutPromise(interval).then(() => rekwest(options.url, options).then(resolve, reject));
+          }
+
           return rekwest(options.url, options).then(resolve, reject);
         }
       }
 
-      Reflect.defineProperty(res, 'ok', {
-        enumerable: true,
-        value: /^2\d{2}$/.test(res.statusCode),
-      });
-
-      Reflect.defineProperty(res, 'redirected', {
-        enumerable: true,
-        value: options.redirected,
-      });
-
       if (res.statusCode >= HTTP_STATUS_BAD_REQUEST) {
-        return reject(premix(res, options));
+        return reject(mixin(res, options));
       }
 
-      resolve(premix(res, options));
-    });
-
-    req.on('end', () => {
-      client?.close();
-    });
-    req.on('error', reject);
-    req.on('frameError', reject);
-    req.on('goaway', reject);
-    req.on('timeout', req.destroy);
-    req.on('trailers', (trailers) => {
-      Reflect.defineProperty(req, 'trailers', {
-        enumerable: true,
-        value: trailers,
-      });
+      resolve(mixin(res, options));
     });
 
     dispatch(req, { ...options, body });
@@ -191,6 +205,25 @@ export default async function rekwest(url, options = {}) {
 
     return res;
   } catch (ex) {
+    if (options.retry?.attempts && options.retry?.statusCodes.includes(ex.statusCode)) {
+      let { interval } = options.retry;
+
+      if (options.retry.retryAfter && ex.headers[HTTP2_HEADER_RETRY_AFTER]) {
+        interval = ex.headers[HTTP2_HEADER_RETRY_AFTER];
+        interval = Number(interval) * 1000 || new Date(interval) - Date.now();
+        if (interval > options.maxRetryAfter) {
+          throw maxRetryAfterError(interval, { cause: ex });
+        }
+      } else {
+        interval = new Function('interval', `return Math.ceil(${ options.retry.backoffStrategy });`)(interval);
+      }
+
+      options.retry.attempts--;
+      options.retry.interval = interval;
+
+      return setTimeoutPromise(interval).then(() => rekwest(url, options));
+    }
+
     if (digest && !redirected && ex.body) {
       ex.body = await ex.body();
     }
@@ -211,28 +244,38 @@ Reflect.defineProperty(rekwest, 'stream', {
       ...merge(rekwest.defaults, {
         headers: { [HTTP2_HEADER_CONTENT_TYPE]: APPLICATION_OCTET_STREAM },
       }, options),
+      redirect: redirects.manual,
     });
 
-    if (options.h2) {
-      const client = http2.connect(url.origin, options);
-      const req = client.request(options.headers, options);
-
-      req.on('end', () => {
-        client.close();
-      });
+    const { h2, url: { protocol } } = options;
+    const { request } = (protocol === 'http:' ? http : https);
+    let client, req;
 
-      return req;
+    if (h2) {
+      client = http2.connect(url.origin, options);
+      req = client.request(options.headers, options);
+    } else {
+      req = request(options.url, options);
     }
 
-    const { url: { protocol } } = options;
-    const { request } = (protocol === 'http:' ? http : https);
+    affix(client, req, options);
+    req.once('response', (res) => {
+      let headers;
+
+      if (h2) {
+        headers = res;
+        res = req;
+      }
+
+      admix(res, headers, options);
+    });
 
-    return request(options.url, options);
+    return req;
   },
 });
 
 Reflect.defineProperty(rekwest, 'defaults', {
   enumerable: true,
-  value: Object.create(null),
-  writable: true,
+  get() { return defaults; },
+  set(value) { defaults = merge(defaults, value); },
 });