urllib 3.0.0-alpha.1 → 3.0.0

package/src/Response.ts

@@ -1,21 +1,21 @@
-import { ReadableStream } from 'stream/web';
 import { Readable } from 'stream';
+import { IncomingHttpHeaders } from 'http';
 
 export type HttpClientResponseMeta = {
   status: number;
   statusCode: number;
-  statusMessage: string;
-  headers: Record<string, string>;
+  headers: IncomingHttpHeaders;
   size: number;
   aborted: boolean;
   rt: number;
   keepAliveSocket: boolean;
-  requestUrls: string[],
+  requestUrls: string[];
   /**
    * https://eggjs.org/en/core/httpclient.html#timing-boolean
    */
   timing: {
     contentDownload: number;
+    waiting: number;
   };
   // remoteAddress: remoteAddress,
   // remotePort: remotePort,
@@ -23,18 +23,19 @@ export type HttpClientResponseMeta = {
   // socketHandledResponses: socketHandledResponses,
 };
 
-export type ReadableStreamWithMeta = (Readable | ReadableStream) & {
+export type ReadableWithMeta = Readable & {
   status: number;
   statusCode: number;
-  statusMessage: string;
-  headers: Record<string, string>;
+  headers: IncomingHttpHeaders;
 };
 
 export type HttpClientResponse = {
-  data: any;
+  opaque: unknown;
+  data: any
   status: number;
-  headers: Record<string, string>;
+  headers: IncomingHttpHeaders;
   url: string;
   redirected: boolean;
-  res: ReadableStreamWithMeta | HttpClientResponseMeta;
+  requestUrls: string[];
+  res: ReadableWithMeta | HttpClientResponseMeta;
 };

package/src/cjs/HttpAgent.d.ts

@@ -0,0 +1,16 @@
+/// <reference types="node" />
+import { LookupFunction } from 'net';
+import { Agent } from 'undici';
+import { DispatchHandlers } from 'undici/types/dispatcher';
+import { BuildOptions } from 'undici/types/connector';
+export declare type CheckAddressFunction = (ip: string, family: number | string) => boolean;
+export declare type HttpAgentOptions = {
+    lookup?: LookupFunction;
+    checkAddress?: CheckAddressFunction;
+    connect?: BuildOptions;
+};
+export declare class HttpAgent extends Agent {
+    #private;
+    constructor(options: HttpAgentOptions);
+    dispatch(options: Agent.DispatchOptions, handler: DispatchHandlers): boolean;
+}

package/src/cjs/HttpAgent.js

@@ -0,0 +1,62 @@
+"use strict";
+var _HttpAgent_checkAddress;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HttpAgent = void 0;
+const tslib_1 = require("tslib");
+const dns_1 = tslib_1.__importDefault(require("dns"));
+const net_1 = require("net");
+const undici_1 = require("undici");
+class IllegalAddressError extends Error {
+    constructor(hostname, ip, family) {
+        const message = 'illegal address';
+        super(message);
+        this.name = this.constructor.name;
+        this.hostname = hostname;
+        this.ip = ip;
+        this.family = family;
+        Error.captureStackTrace(this, this.constructor);
+    }
+}
+class HttpAgent extends undici_1.Agent {
+    constructor(options) {
+        var _a;
+        /* eslint node/prefer-promises/dns: off*/
+        const _lookup = (_a = options.lookup) !== null && _a !== void 0 ? _a : dns_1.default.lookup;
+        const lookup = (hostname, dnsOptions, callback) => {
+            _lookup(hostname, dnsOptions, (err, address, family) => {
+                if (err)
+                    return callback(err, address, family);
+                if (options.checkAddress && !options.checkAddress(address, family)) {
+                    err = new IllegalAddressError(hostname, address, family);
+                }
+                callback(err, address, family);
+            });
+        };
+        super({
+            connect: { ...options.connect, lookup },
+        });
+        _HttpAgent_checkAddress.set(this, void 0);
+        tslib_1.__classPrivateFieldSet(this, _HttpAgent_checkAddress, options.checkAddress, "f");
+    }
+    dispatch(options, handler) {
+        if (tslib_1.__classPrivateFieldGet(this, _HttpAgent_checkAddress, "f") && options.origin) {
+            const originUrl = typeof options.origin === 'string' ? new URL(options.origin) : options.origin;
+            let hostname = originUrl.hostname;
+            // [2001:db8:2de::e13] => 2001:db8:2de::e13
+            if (hostname.startsWith('[') && hostname.endsWith(']')) {
+                hostname = hostname.substring(1, hostname.length - 1);
+            }
+            const family = (0, net_1.isIP)(hostname);
+            if (family === 4 || family === 6) {
+                // if request hostname is ip, custom lookup won't excute
+                if (!tslib_1.__classPrivateFieldGet(this, _HttpAgent_checkAddress, "f").call(this, hostname, family)) {
+                    throw new IllegalAddressError(hostname, hostname, family);
+                }
+            }
+        }
+        return super.dispatch(options, handler);
+    }
+}
+exports.HttpAgent = HttpAgent;
+_HttpAgent_checkAddress = new WeakMap();
+//# sourceMappingURL=HttpAgent.js.map

package/src/cjs/HttpAgent.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"HttpAgent.js","sourceRoot":"","sources":["../HttpAgent.ts"],"names":[],"mappings":";;;;;AAAA,sDAAsB;AACtB,6BAA2C;AAC3C,mCAEgB;AAYhB,MAAM,mBAAoB,SAAQ,KAAK;IAKrC,YAAY,QAAgB,EAAE,EAAU,EAAE,MAAc;QACtD,MAAM,OAAO,GAAG,iBAAiB,CAAC;QAClC,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;QAClC,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,KAAK,CAAC,iBAAiB,CAAC,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IAClD,CAAC;CACF;AAED,MAAa,SAAU,SAAQ,cAAK;IAGlC,YAAY,OAAyB;;QACnC,yCAAyC;QACzC,MAAM,OAAO,GAAG,MAAA,OAAO,CAAC,MAAM,mCAAI,aAAG,CAAC,MAAM,CAAC;QAC7C,MAAM,MAAM,GAAmB,CAAC,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,EAAE;YAChE,OAAO,CAAC,QAAQ,EAAE,UAAU,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE;gBACrD,IAAI,GAAG;oBAAE,OAAO,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;gBAC/C,IAAI,OAAO,CAAC,YAAY,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE;oBAClE,GAAG,GAAG,IAAI,mBAAmB,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;iBAC1D;gBACD,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;YACjC,CAAC,CAAC,CAAC;QACL,CAAC,CAAC;QACF,KAAK,CAAC;YACJ,OAAO,EAAE,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE;SACxC,CAAC,CAAC;QAhBL,0CAAqC;QAiBnC,+BAAA,IAAI,2BAAiB,OAAO,CAAC,YAAY,MAAA,CAAC;IAC5C,CAAC;IAED,QAAQ,CAAC,OAA8B,EAAE,OAAyB;QAChE,IAAI,+BAAA,IAAI,+BAAc,IAAI,OAAO,CAAC,MAAM,EAAE;YACxC,MAAM,SAAS,GAAG,OAAO,OAAO,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;YAChG,IAAI,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAC;YAClC,2CAA2C;YAC3C,IAAI,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;gBACtD,QAAQ,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;aACvD;YACD,MAAM,MAAM,GAAG,IAAA,UAAI,EAAC,QAAQ,CAAC,CAAC;YAC9B,IAAI,MAAM,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC,EAAE;gBAChC,wDAAwD;gBACxD,IAAI,CAAC,+BAAA,IAAI,+BAAc,MAAlB,IAAI,EAAe,QAAQ,EAAE,MAAM,CAAC,EAAE;oBACzC,MAAM,IAAI,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;iBAC3D;aACF;SACF;QACD,OAAO,KAAK,CAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC1C,CAAC;CACF;AAvCD,8BAuCC"}

package/src/cjs/HttpClient.d.ts

@@ -1,12 +1,39 @@
 /// <reference types="node" />
+/// <reference types="node" />
+/// <reference types="node" />
 import { EventEmitter } from 'events';
+import { LookupFunction } from 'net';
+import { CheckAddressFunction } from './HttpAgent';
 import { RequestURL, RequestOptions } from './Request';
 import { HttpClientResponse } from './Response';
 export declare type ClientOptions = {
     defaultArgs?: RequestOptions;
+    /**
+     * Custom DNS lookup function, default is `dns.lookup`.
+     */
+    lookup?: LookupFunction;
+    /**
+      * check request address to protect from SSRF and similar attacks.
+      * It receive two arguments(ip and family) and should return true or false to identified the address is legal or not.
+      * It rely on lookup and have the same version requirement.
+      */
+    checkAddress?: CheckAddressFunction;
+    connect?: {
+        key?: string | Buffer;
+        /**
+        * A string or Buffer containing the certificate key of the client in PEM format.
+        * Notes: This is necessary only if using the client certificate authentication
+        */
+        cert?: string | Buffer;
+        /**
+        * If true, the server certificate is verified against the list of supplied CAs.
+        * An 'error' event is emitted if verification fails.Default: true.
+        */
+        rejectUnauthorized?: boolean;
+    };
 };
 export declare class HttpClient extends EventEmitter {
-    defaultArgs?: RequestOptions;
+    #private;
     constructor(clientOptions?: ClientOptions);
     request(url: RequestURL, options?: RequestOptions): Promise<HttpClientResponse>;
 }