urllib 3.0.0-alpha.0 → 3.0.1

package/src/Request.ts

@@ -1,20 +1,31 @@
 import { Readable, Writable } from 'stream';
-import { LookupFunction } from 'net';
+import { IncomingHttpHeaders } from 'http';
+import type {
+  HttpMethod as UndiciHttpMethod,
+} from 'undici/types/dispatcher';
+import type {
+  HttpClientResponse,
+} from './Response';
 
-export type HttpMethod = 'GET' | 'POST' | 'DELETE' | 'PUT' | 'HEAD' | 'OPTIONS' | 'PATCH' | 'TRACE' | 'CONNECT';
+export type HttpMethod = UndiciHttpMethod;
 
 export type RequestURL = string | URL;
 
+export type FixJSONCtlCharsHandler = (data: string) => string;
+export type FixJSONCtlChars = boolean | FixJSONCtlCharsHandler;
+
 export type RequestOptions = {
   /** Request method, defaults to GET. Could be GET, POST, DELETE or PUT. Alias 'type'. */
   method?: HttpMethod | Lowercase<HttpMethod>;
   /** Data to be sent. Will be stringify automatically. */
   data?: any;
-  /** Force convert data to query string. */
-  dataAsQueryString?: boolean;
   /** Manually set the content of payload. If set, data will be ignored. */
   content?: string | Buffer | Readable;
-  /** Stream to be pipe to the remote. If set, data and content will be ignored. */
+  /**
+   * @deprecated
+   * Stream to be pipe to the remote. If set, data and content will be ignored.
+   * Alias to `content = Readable`
+   */
   stream?: Readable;
   /**
    * A writable stream to be piped by the response stream.
@@ -22,8 +33,6 @@ export type RequestOptions = {
    * will be called with data set null after finished writing.
    */
   writeStream?: Writable;
-  /** consume the writeStream, invoke the callback after writeStream close. */
-  consumeWriteStream?: boolean;
   /**
     * The files will send with multipart/form-data format, base on formstream.
     * If method not set, will use POST method by default.
@@ -36,19 +45,25 @@ export type RequestOptions = {
    * If it's text, the callbacked data would be a String.
    * If it's json, the data of callback would be a parsed JSON Object
    * and will auto set Accept: 'application/json' header.
-   * Default is buffer.
+   * Default is 'buffer'.
    */
   dataType?: 'text' | 'json' | 'buffer' | 'stream';
   /**
+   * @deprecated
    * Let you get the res object when request connected, default false.
    * If set to true, `data` will be response readable stream.
-   * Equal to `dataType = 'stream'`
+   * Alias to `dataType = 'stream'`
    */
   streaming?: boolean;
+  /**
+   * @deprecated
+   * Alias to `dataType = 'stream'`
+   */
+  customResponse?: boolean;
   /** Fix the control characters (U+0000 through U+001F) before JSON parse response. Default is false. */
-  fixJSONCtlChars?: boolean;
+  fixJSONCtlChars?: FixJSONCtlChars;
   /** Request headers. */
-  headers?: Record<string, string>;
+  headers?: IncomingHttpHeaders;
   /**
    * Request timeout in milliseconds for connecting phase and response receiving phase.
    * Defaults to exports.
@@ -56,41 +71,12 @@ export type RequestOptions = {
    * timeout: [3000, 5000], which will set connecting timeout to 3s and response 5s.
    */
   timeout?: number | number[];
-  /** username:password used in HTTP Basic Authorization. */
-  auth?: string;
-  /** username:password used in HTTP Digest Authorization. */
-  digestAuth?: string;
-  /**
-   * An array of strings or Buffers of trusted certificates.
-   * If this is omitted several well known "root" CAs will be used, like VeriSign.
-   * These are used to authorize connections.
-   * Notes: This is necessary only if the server uses the self - signed certificate
-   */
-  ca?: string | Buffer | string[] | Buffer[];
   /**
-   * If true, the server certificate is verified against the list of supplied CAs.
-   * An 'error' event is emitted if verification fails.Default: true.
-   */
-  rejectUnauthorized?: boolean;
-  /** A string or Buffer containing the private key, certificate and CA certs of the server in PFX or PKCS12 format. */
-  pfx?: string | Buffer;
-  /**
-   * A string or Buffer containing the private key of the client in PEM format.
-   * Notes: This is necessary only if using the client certificate authentication
-   */
-  key?: string | Buffer;
-  /**
-   * A string or Buffer containing the certificate key of the client in PEM format.
-   * Notes: This is necessary only if using the client certificate authentication
-   */
-  cert?: string | Buffer;
-  /** A string of passphrase for the private key or pfx. */
-  passphrase?: string;
-  /** A string describing the ciphers to use or exclude. */
-  ciphers?: string;
-  /** The SSL method to use, e.g.SSLv3_method to force SSL version 3. */
-  secureProtocol?: string;
-  /** follow HTTP 3xx responses as redirects. defaults to false. */
+   * username:password used in HTTP Basic Authorization.
+   * Alias to `headers.authorization = xxx`
+   **/
+  auth?: string;
+  /** follow HTTP 3xx responses as redirects. defaults to true. */
   followRedirect?: boolean;
   /** The maximum number of redirects to follow, defaults to 10. */
   maxRedirects?: number;
@@ -98,19 +84,35 @@ export type RequestOptions = {
   formatRedirectUrl?: (a: any, b: any) => void;
   /** Before request hook, you can change every thing here. */
   beforeRequest?: (...args: any[]) => void;
-  /** Accept gzip response content and auto decode it, default is false. */
+  /** Accept `gzip, br` response content and auto decode it, default is false. */
+  compressed?: boolean;
+  /**
+   * @deprecated
+   * Alias to compressed
+   * */
   gzip?: boolean;
-  /** Enable timing or not, default is false. */
+  /**
+   * @deprecated
+   * Enable timing or not, default is false.
+   * */
   timing?: boolean;
   /**
-   * Custom DNS lookup function, default is dns.lookup.
-   * Require node >= 4.0.0(for http protocol) and node >=8(for https protocol)
+   * Auto retry times on 5xx response, default is 0. Don't work on streaming request
+   * It's not supported by using retry and writeStream, because the retry request can't stop the stream which is consuming.
+   **/
+  retry?: number;
+  /** Wait a delay(ms) between retries */
+  retryDelay?: number;
+  /**
+   * Determine whether retry, a response object as the first argument.
+   * It will retry when status >= 500 by default. Request error is not included.
    */
-  lookup?: LookupFunction;
+  isRetry?: (response: HttpClientResponse) => boolean;
+  /** Default: `null` */
+  opaque?: unknown;
   /**
-   * check request address to protect from SSRF and similar attacks.
-   * It receive two arguments(ip and family) and should return true or false to identified the address is legal or not.
-   * It rely on lookup and have the same version requirement.
+   * @deprecated
+   * Maybe you should use opaque instead
    */
-  checkAddress?: (ip: string, family: number | string) => boolean;
+  ctx?: unknown;
 };

package/src/Response.ts

@@ -1,20 +1,41 @@
-import { OutgoingHttpHeaders, IncomingMessage } from 'http';
+import { Readable } from 'stream';
+import { IncomingHttpHeaders } from 'http';
 
-export interface HttpClientResponse<T> {
-  data: T;
+export type HttpClientResponseMeta = {
   status: number;
-  headers: OutgoingHttpHeaders;
-  res: IncomingMessage & {
-    /**
-     * https://eggjs.org/en/core/httpclient.html#timing-boolean
-     */
-    timing?: {
-      queuing: number;
-      dnslookup: number;
-      connected: number;
-      requestSent: number;
-      waiting: number;
-      contentDownload: number;
-    }
+  statusCode: number;
+  headers: IncomingHttpHeaders;
+  size: number;
+  aborted: boolean;
+  rt: number;
+  keepAliveSocket: boolean;
+  requestUrls: string[];
+  /**
+   * https://eggjs.org/en/core/httpclient.html#timing-boolean
+   */
+  timing: {
+    contentDownload: number;
+    waiting: number;
   };
-}
+  // remoteAddress: remoteAddress,
+  // remotePort: remotePort,
+  // socketHandledRequests: socketHandledRequests,
+  // socketHandledResponses: socketHandledResponses,
+};
+
+export type ReadableWithMeta = Readable & {
+  status: number;
+  statusCode: number;
+  headers: IncomingHttpHeaders;
+};
+
+export type HttpClientResponse = {
+  opaque: unknown;
+  data: any
+  status: number;
+  headers: IncomingHttpHeaders;
+  url: string;
+  redirected: boolean;
+  requestUrls: string[];
+  res: ReadableWithMeta | HttpClientResponseMeta;
+};

package/src/cjs/HttpAgent.d.ts

@@ -0,0 +1,16 @@
+/// <reference types="node" />
+import { LookupFunction } from 'net';
+import { Agent } from 'undici';
+import { DispatchHandlers } from 'undici/types/dispatcher';
+import { BuildOptions } from 'undici/types/connector';
+export declare type CheckAddressFunction = (ip: string, family: number | string) => boolean;
+export declare type HttpAgentOptions = {
+    lookup?: LookupFunction;
+    checkAddress?: CheckAddressFunction;
+    connect?: BuildOptions;
+};
+export declare class HttpAgent extends Agent {
+    #private;
+    constructor(options: HttpAgentOptions);
+    dispatch(options: Agent.DispatchOptions, handler: DispatchHandlers): boolean;
+}

package/src/cjs/HttpAgent.js

@@ -0,0 +1,62 @@
+"use strict";
+var _HttpAgent_checkAddress;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HttpAgent = void 0;
+const tslib_1 = require("tslib");
+const dns_1 = tslib_1.__importDefault(require("dns"));
+const net_1 = require("net");
+const undici_1 = require("undici");
+class IllegalAddressError extends Error {
+    constructor(hostname, ip, family) {
+        const message = 'illegal address';
+        super(message);
+        this.name = this.constructor.name;
+        this.hostname = hostname;
+        this.ip = ip;
+        this.family = family;
+        Error.captureStackTrace(this, this.constructor);
+    }
+}
+class HttpAgent extends undici_1.Agent {
+    constructor(options) {
+        var _a;
+        /* eslint node/prefer-promises/dns: off*/
+        const _lookup = (_a = options.lookup) !== null && _a !== void 0 ? _a : dns_1.default.lookup;
+        const lookup = (hostname, dnsOptions, callback) => {
+            _lookup(hostname, dnsOptions, (err, address, family) => {
+                if (err)
+                    return callback(err, address, family);
+                if (options.checkAddress && !options.checkAddress(address, family)) {
+                    err = new IllegalAddressError(hostname, address, family);
+                }
+                callback(err, address, family);
+            });
+        };
+        super({
+            connect: { ...options.connect, lookup },
+        });
+        _HttpAgent_checkAddress.set(this, void 0);
+        tslib_1.__classPrivateFieldSet(this, _HttpAgent_checkAddress, options.checkAddress, "f");
+    }
+    dispatch(options, handler) {
+        if (tslib_1.__classPrivateFieldGet(this, _HttpAgent_checkAddress, "f") && options.origin) {
+            const originUrl = typeof options.origin === 'string' ? new URL(options.origin) : options.origin;
+            let hostname = originUrl.hostname;
+            // [2001:db8:2de::e13] => 2001:db8:2de::e13
+            if (hostname.startsWith('[') && hostname.endsWith(']')) {
+                hostname = hostname.substring(1, hostname.length - 1);
+            }
+            const family = (0, net_1.isIP)(hostname);
+            if (family === 4 || family === 6) {
+                // if request hostname is ip, custom lookup won't excute
+                if (!tslib_1.__classPrivateFieldGet(this, _HttpAgent_checkAddress, "f").call(this, hostname, family)) {
+                    throw new IllegalAddressError(hostname, hostname, family);
+                }
+            }
+        }
+        return super.dispatch(options, handler);
+    }
+}
+exports.HttpAgent = HttpAgent;
+_HttpAgent_checkAddress = new WeakMap();
+//# sourceMappingURL=HttpAgent.js.map

package/src/cjs/HttpAgent.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"HttpAgent.js","sourceRoot":"","sources":["../HttpAgent.ts"],"names":[],"mappings":";;;;;AAAA,sDAAsB;AACtB,6BAA2C;AAC3C,mCAEgB;AAYhB,MAAM,mBAAoB,SAAQ,KAAK;IAKrC,YAAY,QAAgB,EAAE,EAAU,EAAE,MAAc;QACtD,MAAM,OAAO,GAAG,iBAAiB,CAAC;QAClC,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;QAClC,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,KAAK,CAAC,iBAAiB,CAAC,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IAClD,CAAC;CACF;AAED,MAAa,SAAU,SAAQ,cAAK;IAGlC,YAAY,OAAyB;;QACnC,yCAAyC;QACzC,MAAM,OAAO,GAAG,MAAA,OAAO,CAAC,MAAM,mCAAI,aAAG,CAAC,MAAM,CAAC;QAC7C,MAAM,MAAM,GAAmB,CAAC,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,EAAE;YAChE,OAAO,CAAC,QAAQ,EAAE,UAAU,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE;gBACrD,IAAI,GAAG;oBAAE,OAAO,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;gBAC/C,IAAI,OAAO,CAAC,YAAY,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE;oBAClE,GAAG,GAAG,IAAI,mBAAmB,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;iBAC1D;gBACD,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;YACjC,CAAC,CAAC,CAAC;QACL,CAAC,CAAC;QACF,KAAK,CAAC;YACJ,OAAO,EAAE,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE;SACxC,CAAC,CAAC;QAhBL,0CAAqC;QAiBnC,+BAAA,IAAI,2BAAiB,OAAO,CAAC,YAAY,MAAA,CAAC;IAC5C,CAAC;IAED,QAAQ,CAAC,OAA8B,EAAE,OAAyB;QAChE,IAAI,+BAAA,IAAI,+BAAc,IAAI,OAAO,CAAC,MAAM,EAAE;YACxC,MAAM,SAAS,GAAG,OAAO,OAAO,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;YAChG,IAAI,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAC;YAClC,2CAA2C;YAC3C,IAAI,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;gBACtD,QAAQ,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;aACvD;YACD,MAAM,MAAM,GAAG,IAAA,UAAI,EAAC,QAAQ,CAAC,CAAC;YAC9B,IAAI,MAAM,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC,EAAE;gBAChC,wDAAwD;gBACxD,IAAI,CAAC,+BAAA,IAAI,+BAAc,MAAlB,IAAI,EAAe,QAAQ,EAAE,MAAM,CAAC,EAAE;oBACzC,MAAM,IAAI,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;iBAC3D;aACF;SACF;QACD,OAAO,KAAK,CAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC1C,CAAC;CACF;AAvCD,8BAuCC"}

package/src/cjs/HttpClient.d.ts

@@ -1,31 +1,40 @@
 /// <reference types="node" />
+/// <reference types="node" />
+/// <reference types="node" />
 import { EventEmitter } from 'events';
+import { LookupFunction } from 'net';
+import { CheckAddressFunction } from './HttpAgent';
 import { RequestURL, RequestOptions } from './Request';
+import { HttpClientResponse } from './Response';
 export declare type ClientOptions = {
     defaultArgs?: RequestOptions;
+    /**
+     * Custom DNS lookup function, default is `dns.lookup`.
+     */
+    lookup?: LookupFunction;
+    /**
+      * check request address to protect from SSRF and similar attacks.
+      * It receive two arguments(ip and family) and should return true or false to identified the address is legal or not.
+      * It rely on lookup and have the same version requirement.
+      */
+    checkAddress?: CheckAddressFunction;
+    connect?: {
+        key?: string | Buffer;
+        /**
+        * A string or Buffer containing the certificate key of the client in PEM format.
+        * Notes: This is necessary only if using the client certificate authentication
+        */
+        cert?: string | Buffer;
+        /**
+        * If true, the server certificate is verified against the list of supplied CAs.
+        * An 'error' event is emitted if verification fails.Default: true.
+        */
+        rejectUnauthorized?: boolean;
+    };
 };
+export declare const HEADER_USER_AGENT: string;
 export declare class HttpClient extends EventEmitter {
-    defaultArgs?: RequestOptions;
+    #private;
     constructor(clientOptions?: ClientOptions);
-    request(url: RequestURL, options?: RequestOptions): Promise<{
-        status: number;
-        data: any;
-        headers: Record<string, string>;
-        url: string;
-        redirected: boolean;
-        res: {
-            status: number;
-            statusCode: number;
-            statusMessage: string;
-            headers: Record<string, string>;
-            size: number;
-            aborted: boolean;
-            rt: number;
-            keepAliveSocket: boolean;
-            requestUrls: string[];
-            timing: {
-                contentDownload: number;
-            };
-        };
-    }>;
+    request(url: RequestURL, options?: RequestOptions): Promise<HttpClientResponse>;
 }