urllib 3.0.0-alpha.1 → 3.0.2

package/src/HttpClient.ts

@@ -1,25 +1,87 @@
 import { EventEmitter } from 'events';
+import { LookupFunction } from 'net';
 import { debuglog } from 'util';
-import { Readable, isReadable } from 'stream';
-import { pipeline } from 'stream/promises';
+import {
+  createGunzip,
+  createBrotliDecompress,
+  gunzipSync,
+  brotliDecompressSync,
+} from 'zlib';
 import { Blob } from 'buffer';
-import { createReadStream } from 'fs';
+import { Readable, pipeline } from 'stream';
+import stream from 'stream';
 import { basename } from 'path';
+import { createReadStream } from 'fs';
+import { IncomingHttpHeaders } from 'http';
+import { performance } from 'perf_hooks';
 import {
-  fetch, RequestInit, Headers, FormData,
+  FormData as FormDataNative,
+  request as undiciRequest,
+  Dispatcher,
 } from 'undici';
+import { FormData as FormDataNode } from 'formdata-node';
+import { FormDataEncoder } from 'form-data-encoder';
 import createUserAgent from 'default-user-agent';
 import mime from 'mime-types';
-import { RequestURL, RequestOptions } from './Request';
-import { HttpClientResponseMeta, HttpClientResponse, ReadableStreamWithMeta } from './Response';
-import { parseJSON } from './utils';
+import pump from 'pump';
+import { HttpAgent, CheckAddressFunction } from './HttpAgent';
+import { RequestURL, RequestOptions, HttpMethod } from './Request';
+import { HttpClientResponseMeta, HttpClientResponse, ReadableWithMeta } from './Response';
+import { parseJSON, sleep } from './utils';
+
+const FormData = FormDataNative ?? FormDataNode;
+// impl isReadable on Node.js 14
+const isReadable = stream.isReadable ?? function isReadable(stream: any) {
+  return stream && typeof stream.read === 'function';
+};
+// impl promise pipeline on Node.js 14
+const pipelinePromise = stream.promises?.pipeline ?? function pipeline(...args: any[]) {
+  return new Promise<void>((resolve, reject) => {
+    pump(...args, (err?: Error) => {
+      if (err) return reject(err);
+      resolve();
+    });
+  });
+};
+
+function noop() {
+  // noop
+}
+
+const MAX_REQURE_ID_VALUE = Math.pow(2, 31) - 10;
+let globalRequestId = 0;
 
 const debug = debuglog('urllib');
 
 export type ClientOptions = {
   defaultArgs?: RequestOptions;
+  /**
+   * Custom DNS lookup function, default is `dns.lookup`.
+   */
+  lookup?: LookupFunction;
+  /**
+    * check request address to protect from SSRF and similar attacks.
+    * It receive two arguments(ip and family) and should return true or false to identified the address is legal or not.
+    * It rely on lookup and have the same version requirement.
+    */
+  checkAddress?: CheckAddressFunction;
+  connect?: {
+    key?: string | Buffer;
+    /**
+    * A string or Buffer containing the certificate key of the client in PEM format.
+    * Notes: This is necessary only if using the client certificate authentication
+    */
+    cert?: string | Buffer;
+    /**
+    * If true, the server certificate is verified against the list of supplied CAs.
+    * An 'error' event is emitted if verification fails.Default: true.
+    */
+    rejectUnauthorized?: boolean;
+  },
 };
 
+type UndiciRquestOptions = { dispatcher?: Dispatcher } & Omit<Dispatcher.RequestOptions, 'origin' | 'path' | 'method'> & Partial<Pick<Dispatcher.RequestOptions, 'method'>>;
+
 // https://github.com/octet-stream/form-data
 class BlobFromStream {
   #stream;
@@ -51,7 +113,7 @@ class HttpClientRequestTimeoutError extends Error {
   }
 }
 
-const HEADER_USER_AGENT = createUserAgent('node-urllib', '3.0.0');
+export const HEADER_USER_AGENT = createUserAgent('node-urllib', '3.0.0');
 
 function getFileName(stream: Readable) {
   const filePath: string = (stream as any).path;
@@ -61,69 +123,140 @@ function getFileName(stream: Readable) {
   return '';
 }
 
+function defaultIsRetry(response: HttpClientResponse) {
+  return response.status >= 500;
+}
+
+function performanceTime(startTime: number) {
+  return Math.floor((performance.now() - startTime) * 1000) / 1000;
+}
+
+type RequestContext = {
+  retries: number;
+};
+
 export class HttpClient extends EventEmitter {
-  defaultArgs?: RequestOptions;
+  #defaultArgs?: RequestOptions;
+  #dispatcher?: Dispatcher;
 
   constructor(clientOptions?: ClientOptions) {
     super();
-    this.defaultArgs = clientOptions?.defaultArgs;
+    this.#defaultArgs = clientOptions?.defaultArgs;
+    if (clientOptions?.lookup || clientOptions?.checkAddress || clientOptions?.connect) {
+      this.#dispatcher = new HttpAgent({
+        lookup: clientOptions.lookup,
+        checkAddress: clientOptions.checkAddress,
+        connect: clientOptions.connect,
+      });
+    }
   }
 
   async request(url: RequestURL, options?: RequestOptions) {
+    return await this.#requestInternal(url, options);
+  }
+
+  async #requestInternal(url: RequestURL, options?: RequestOptions, requestContext?: RequestContext): Promise<HttpClientResponse> {
+    if (globalRequestId >= MAX_REQURE_ID_VALUE) {
+      globalRequestId = 0;
+    }
+    const requestId = ++globalRequestId;
+
     const requestUrl = typeof url === 'string' ? new URL(url) : url;
     const args = {
-      ...this.defaultArgs,
+      retry: 0,
+      ...this.#defaultArgs,
       ...options,
-      emitter: this,
     };
-    const requestStartTime = Date.now();
+    requestContext = {
+      retries: 0,
+      ...requestContext,
+    };
+    const requestStartTime = performance.now();
+
+    const reqMeta = {
+      requestId,
+      url: requestUrl.href,
+      args,
+      ctx: args.ctx,
+    };
     // keep urllib createCallbackResponse style
-    const resHeaders: Record<string, string> = {};
+    const resHeaders: IncomingHttpHeaders = {};
     const res: HttpClientResponseMeta = {
       status: -1,
       statusCode: -1,
-      statusMessage: '',
       headers: resHeaders,
       size: 0,
       aborted: false,
       rt: 0,
       keepAliveSocket: true,
-      requestUrls: [ url.toString() ],
+      requestUrls: [],
       timing: {
+        waiting: 0,
         contentDownload: 0,
       },
     };
 
-    let requestTimeout = 5000;
+    let headersTimeout = 5000;
+    let bodyTimeout = 5000;
     if (args.timeout) {
       if (Array.isArray(args.timeout)) {
-        requestTimeout = args.timeout[args.timeout.length - 1] ?? requestTimeout;
+        headersTimeout = args.timeout[0] ?? headersTimeout;
+        bodyTimeout = args.timeout[1] ?? bodyTimeout;
       } else {
-        requestTimeout = args.timeout;
+        headersTimeout = bodyTimeout = args.timeout;
       }
     }
 
-    const requestTimeoutController = new AbortController();
-    const requestTimerId = setTimeout(() => requestTimeoutController.abort(), requestTimeout);
-    const method = (args.method ?? 'GET').toUpperCase();
-
-    try {
-      const headers = new Headers(args.headers ?? {});
-      if (!headers.has('user-agent')) {
-        // need to set user-agent
-        headers.set('user-agent', HEADER_USER_AGENT);
-      }
-      if (args.dataType === 'json' && !headers.has('accept')) {
-        headers.set('accept', 'application/json');
+    const method = (args.method ?? 'GET').toUpperCase() as HttpMethod;
+    const headers: IncomingHttpHeaders = {};
+    if (args.headers) {
+      // convert headers to lower-case
+      for (const name in args.headers) {
+        headers[name.toLowerCase()] = args.headers[name];
       }
+    }
+    // hidden user-agent
+    const hiddenUserAgent = 'user-agent' in headers && !headers['user-agent'];
+    if (hiddenUserAgent) {
+      delete headers['user-agent'];
+    } else if (!headers['user-agent']) {
+      // need to set user-agent
+      headers['user-agent'] = HEADER_USER_AGENT;
+    }
+    // Alias to dataType = 'stream'
+    if (args.streaming || args.customResponse) {
+      args.dataType = 'stream';
+    }
+    if (args.dataType === 'json' && !headers.accept) {
+      headers.accept = 'application/json';
+    }
+    // gzip alias to compressed
+    if (args.gzip && args.compressed !== false) {
+      args.compressed = true;
+    }
+    if (args.compressed && !headers['accept-encoding']) {
+      headers['accept-encoding'] = 'gzip, br';
+    }
+    if (requestContext.retries > 0) {
+      headers['x-urllib-retry'] = `${requestContext.retries}/${args.retry}`;
+    }
+    if (args.auth && !headers.authorization) {
+      headers.authorization = `Basic ${Buffer.from(args.auth).toString('base64')}`;
+    }
 
-      const requestOptions: RequestInit = {
+    let opaque = args.opaque;
+    try {
+      const requestOptions: UndiciRquestOptions = {
         method,
         keepalive: true,
-        signal: requestTimeoutController.signal,
+        maxRedirections: args.maxRedirects ?? 10,
+        headersTimeout,
+        bodyTimeout,
+        opaque,
+        dispatcher: this.#dispatcher,
       };
       if (args.followRedirect === false) {
-        requestOptions.redirect = 'manual';
+        requestOptions.maxRedirections = 0;
       }
 
       const isGETOrHEAD = requestOptions.method === 'GET' || requestOptions.method === 'HEAD';
@@ -164,8 +297,8 @@ export class HttpClient extends EventEmitter {
             // const fileName = encodeURIComponent(basename(file));
             // formData.append(field, await fileFromPath(file, `utf-8''${fileName}`, { type: mime.lookup(fileName) || '' }));
             const fileName = basename(file);
-            const fileReader = createReadStream(file);
-            formData.append(field, new BlobFromStream(fileReader, mime.lookup(fileName) || ''), fileName);
+            const fileReadable = createReadStream(file);
+            formData.append(field, new BlobFromStream(fileReadable, mime.lookup(fileName) || ''), fileName);
           } else if (Buffer.isBuffer(file)) {
             formData.append(field, new Blob([ file ]), `bufferfile${index}`);
           } else if (file instanceof Readable || isReadable(file as any)) {
@@ -173,17 +306,27 @@ export class HttpClient extends EventEmitter {
             formData.append(field, new BlobFromStream(file, mime.lookup(fileName) || ''), fileName);
           }
         }
-        requestOptions.body = formData;
+
+        if (FormDataNative) {
+          requestOptions.body = formData;
+        } else {
+          // Node.js 14 does not support spec-compliant FormData
+          // https://github.com/octet-stream/form-data#usage
+          const encoder = new FormDataEncoder(formData as any);
+          Object.assign(headers, encoder.headers);
+          // fix "Content-Length":"NaN"
+          delete headers['Content-Length'];
+          requestOptions.body = Readable.from(encoder);
+        }
       } else if (args.content) {
         if (!isGETOrHEAD) {
-          if (isReadable(args.content as Readable)) {
-            // disable keepalive
-            requestOptions.keepalive = false;
-          }
           // handle content
           requestOptions.body = args.content;
           if (args.contentType) {
-            headers.set('content-type', args.contentType);
+            headers['content-type'] = args.contentType;
+          }
+          if (typeof args.content === 'string' && !headers['content-type']) {
+            headers['content-type'] = 'text/plain;charset=UTF-8';
           }
         }
       } else if (args.data) {
@@ -198,98 +341,174 @@ export class HttpClient extends EventEmitter {
           }
         } else {
           if (isStringOrBufferOrReadable) {
-            if (isReadable(args.data as Readable)) {
-              // disable keepalive
-              requestOptions.keepalive = false;
-            }
             requestOptions.body = args.data;
           } else {
             if (args.contentType === 'json'
               || args.contentType === 'application/json'
-              || headers.get('content-type')?.startsWith('application/json')) {
+              || headers['content-type']?.startsWith('application/json')) {
               requestOptions.body = JSON.stringify(args.data);
-              if (!headers.has('content-type')) {
-                headers.set('content-type', 'application/json');
+              if (!headers['content-type']) {
+                headers['content-type'] = 'application/json';
               }
             } else {
-              requestOptions.body = new URLSearchParams(args.data);
+              headers['content-type'] = 'application/x-www-form-urlencoded;charset=UTF-8';
+              requestOptions.body = new URLSearchParams(args.data).toString();
             }
           }
         }
       }
 
-      debug('%s %s, headers: %j, timeout: %s', requestOptions.method, url, headers, requestTimeout);
+      debug('Request#%d %s %s, headers: %j, headersTimeout: %s, bodyTimeout: %s',
+        requestId, requestOptions.method, requestUrl.href, headers, headersTimeout, bodyTimeout);
       requestOptions.headers = headers;
+      if (this.listenerCount('request') > 0) {
+        this.emit('request', reqMeta);
+      }
 
-      const response = await fetch(requestUrl, requestOptions);
-      for (const [ name, value ] of response.headers) {
-        res.headers[name] = value;
+      const response = await undiciRequest(requestUrl, requestOptions);
+      opaque = response.opaque;
+      if (args.timing) {
+        res.timing.waiting = performanceTime(requestStartTime);
       }
-      res.status = res.statusCode = response.status;
-      res.statusMessage = response.statusText;
-      if (response.redirected) {
-        res.requestUrls.push(response.url);
+
+      const context = response.context as { history: URL[] };
+      let lastUrl = '';
+      if (context?.history) {
+        for (const urlObject of context?.history) {
+          res.requestUrls.push(urlObject.href);
+          lastUrl = urlObject.href;
+        }
+      } else {
+        res.requestUrls.push(requestUrl.href);
+        lastUrl = requestUrl.href;
       }
+      const contentEncoding = response.headers['content-encoding'];
+      const isCompressedContent = contentEncoding === 'gzip' || contentEncoding === 'br';
+
+      res.headers = response.headers;
+      res.status = res.statusCode = response.statusCode;
       if (res.headers['content-length']) {
         res.size = parseInt(res.headers['content-length']);
       }
 
       let data: any = null;
-      let responseBodyStream: ReadableStreamWithMeta | undefined;
-      if (args.streaming || args.dataType === 'stream') {
+      let responseBodyStream: ReadableWithMeta | undefined;
+      if (args.dataType === 'stream') {
+        // streaming mode will disable retry
+        args.retry = 0;
         const meta = {
           status: res.status,
           statusCode: res.statusCode,
-          statusMessage: res.statusMessage,
           headers: res.headers,
         };
-        if (typeof Readable.fromWeb === 'function') {
-          responseBodyStream = Object.assign(Readable.fromWeb(response.body!), meta);
+        if (isCompressedContent) {
+          // gzip or br
+          const decoder = contentEncoding === 'gzip' ? createGunzip() : createBrotliDecompress();
+          responseBodyStream = Object.assign(pipeline(response.body, decoder, noop), meta);
         } else {
-          responseBodyStream = Object.assign(response.body!, meta);
+          responseBodyStream = Object.assign(response.body, meta);
         }
       } else if (args.writeStream) {
-        await pipeline(response.body!, args.writeStream);
-      } else if (args.dataType === 'text') {
-        data = await response.text();
-      } else if (args.dataType === 'json') {
-        if (requestOptions.method === 'HEAD') {
-          data = {};
+        // streaming mode will disable retry
+        args.retry = 0;
+        if (isCompressedContent) {
+          const decoder = contentEncoding === 'gzip' ? createGunzip() : createBrotliDecompress();
+          await pipelinePromise(response.body, decoder, args.writeStream);
         } else {
-          data = await response.text();
+          await pipelinePromise(response.body, args.writeStream);
+        }
+      } else {
+        // buffer
+        data = Buffer.from(await response.body.arrayBuffer());
+        if (isCompressedContent) {
+          try {
+            data = contentEncoding === 'gzip' ? gunzipSync(data) : brotliDecompressSync(data);
+          } catch (err: any) {
+            if (err.name === 'Error') {
+              err.name = 'UnzipError';
+            }
+            throw err;
+          }
+        }
+        if (args.dataType === 'text') {
+          data = data.toString();
+        } else if (args.dataType === 'json') {
           if (data.length === 0) {
             data = null;
           } else {
-            data = parseJSON(data, args.fixJSONCtlChars);
+            data = parseJSON(data.toString(), args.fixJSONCtlChars);
           }
         }
-      } else {
-        // buffer
-        data = Buffer.from(await response.arrayBuffer());
       }
-      res.rt = res.timing.contentDownload = Date.now() - requestStartTime;
+      res.rt = performanceTime(requestStartTime);
+      if (args.timing) {
+        res.timing.contentDownload = res.rt;
+      }
 
       const clientResponse: HttpClientResponse = {
-        status: res.status,
+        opaque,
         data,
+        status: res.status,
         headers: res.headers,
-        url: response.url,
-        redirected: response.redirected,
+        url: lastUrl,
+        redirected: res.requestUrls.length > 1,
+        requestUrls: res.requestUrls,
         res: responseBodyStream ?? res,
       };
+
+      if (args.retry > 0 && requestContext.retries < args.retry) {
+        const isRetry = args.isRetry ?? defaultIsRetry;
+        if (isRetry(clientResponse)) {
+          if (args.retryDelay) {
+            await sleep(args.retryDelay);
+          }
+          requestContext.retries++;
+          return await this.#requestInternal(url, options, requestContext);
+        }
+      }
+
+      if (this.listenerCount('response') > 0) {
+        this.emit('response', {
+          requestId,
+          error: null,
+          ctx: args.ctx,
+          req: reqMeta,
+          res,
+        });
+      }
+
       return clientResponse;
     } catch (e: any) {
+      debug('Request#%d throw error: %s', requestId, e);
       let err = e;
-      if (requestTimeoutController.signal.aborted) {
-        err = new HttpClientRequestTimeoutError(requestTimeout, { cause: e });
+      if (err.name === 'HeadersTimeoutError') {
+        err = new HttpClientRequestTimeoutError(headersTimeout, { cause: e });
+      } else if (err.name === 'BodyTimeoutError') {
+        err = new HttpClientRequestTimeoutError(bodyTimeout, { cause: e });
       }
-      err.res = res;
+      err.opaque = opaque;
       err.status = res.status;
       err.headers = res.headers;
-      // console.error(err);
+      err.res = res;
+      // make sure requestUrls not empty
+      if (res.requestUrls.length === 0) {
+        res.requestUrls.push(requestUrl.href);
+      }
+      res.rt = performanceTime(requestStartTime);
+      if (args.timing) {
+        res.timing.contentDownload = res.rt;
+      }
+
+      if (this.listenerCount('response') > 0) {
+        this.emit('response', {
+          requestId,
+          error: err,
+          ctx: args.ctx,
+          req: reqMeta,
+          res,
+        });
+      }
       throw err;
-    } finally {
-      clearTimeout(requestTimerId);
     }
   }
 }