urllib 2.38.1 → 3.0.0

package/src/cjs/Request.d.ts

@@ -0,0 +1,114 @@
+/// <reference types="node" />
+/// <reference types="node" />
+/// <reference types="node" />
+/// <reference types="node" />
+import { Readable, Writable } from 'stream';
+import { IncomingHttpHeaders } from 'http';
+import type { HttpMethod as UndiciHttpMethod } from 'undici/types/dispatcher';
+import type { HttpClientResponse } from './Response';
+export declare type HttpMethod = UndiciHttpMethod;
+export declare type RequestURL = string | URL;
+export declare type FixJSONCtlCharsHandler = (data: string) => string;
+export declare type FixJSONCtlChars = boolean | FixJSONCtlCharsHandler;
+export declare type RequestOptions = {
+    /** Request method, defaults to GET. Could be GET, POST, DELETE or PUT. Alias 'type'. */
+    method?: HttpMethod | Lowercase<HttpMethod>;
+    /** Data to be sent. Will be stringify automatically. */
+    data?: any;
+    /** Manually set the content of payload. If set, data will be ignored. */
+    content?: string | Buffer | Readable;
+    /**
+     * @deprecated
+     * Stream to be pipe to the remote. If set, data and content will be ignored.
+     * Alias to `content = Readable`
+     */
+    stream?: Readable;
+    /**
+     * A writable stream to be piped by the response stream.
+     * Responding data will be write to this stream and callback
+     * will be called with data set null after finished writing.
+     */
+    writeStream?: Writable;
+    /**
+      * The files will send with multipart/form-data format, base on formstream.
+      * If method not set, will use POST method by default.
+      */
+    files?: Array<Readable | Buffer | string> | Record<string, Readable | Buffer | string> | Readable | Buffer | string;
+    /** Type of request data, could be 'json'. If it's 'json', will auto set Content-Type: 'application/json' header. */
+    contentType?: string;
+    /**
+     * Type of response data. Could be text or json.
+     * If it's text, the callbacked data would be a String.
+     * If it's json, the data of callback would be a parsed JSON Object
+     * and will auto set Accept: 'application/json' header.
+     * Default is 'buffer'.
+     */
+    dataType?: 'text' | 'json' | 'buffer' | 'stream';
+    /**
+     * @deprecated
+     * Let you get the res object when request connected, default false.
+     * If set to true, `data` will be response readable stream.
+     * Alias to `dataType = 'stream'`
+     */
+    streaming?: boolean;
+    /**
+     * @deprecated
+     * Alias to `dataType = 'stream'`
+     */
+    customResponse?: boolean;
+    /** Fix the control characters (U+0000 through U+001F) before JSON parse response. Default is false. */
+    fixJSONCtlChars?: FixJSONCtlChars;
+    /** Request headers. */
+    headers?: IncomingHttpHeaders;
+    /**
+     * Request timeout in milliseconds for connecting phase and response receiving phase.
+     * Defaults to exports.
+     * TIMEOUT, both are 5s. You can use timeout: 5000 to tell urllib use same timeout on two phase or set them seperately such as
+     * timeout: [3000, 5000], which will set connecting timeout to 3s and response 5s.
+     */
+    timeout?: number | number[];
+    /**
+     * username:password used in HTTP Basic Authorization.
+     * Alias to `headers.authorization = xxx`
+     **/
+    auth?: string;
+    /** follow HTTP 3xx responses as redirects. defaults to true. */
+    followRedirect?: boolean;
+    /** The maximum number of redirects to follow, defaults to 10. */
+    maxRedirects?: number;
+    /** Format the redirect url by your self. Default is url.resolve(from, to). */
+    formatRedirectUrl?: (a: any, b: any) => void;
+    /** Before request hook, you can change every thing here. */
+    beforeRequest?: (...args: any[]) => void;
+    /** Accept `gzip, br` response content and auto decode it, default is false. */
+    compressed?: boolean;
+    /**
+     * @deprecated
+     * Alias to compressed
+     * */
+    gzip?: boolean;
+    /**
+     * @deprecated
+     * Enable timing or not, default is false.
+     * */
+    timing?: boolean;
+    /**
+     * Auto retry times on 5xx response, default is 0. Don't work on streaming request
+     * It's not supported by using retry and writeStream, because the retry request can't stop the stream which is consuming.
+     **/
+    retry?: number;
+    /** Wait a delay(ms) between retries */
+    retryDelay?: number;
+    /**
+     * Determine whether retry, a response object as the first argument.
+     * It will retry when status >= 500 by default. Request error is not included.
+     */
+    isRetry?: (response: HttpClientResponse) => boolean;
+    /** Default: `null` */
+    opaque?: unknown;
+    /**
+     * @deprecated
+     * Maybe you should use opaque instead
+     */
+    ctx?: unknown;
+};

package/src/cjs/Request.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=Request.js.map

package/src/cjs/Request.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Request.js","sourceRoot":"","sources":["../Request.ts"],"names":[],"mappings":""}

package/src/cjs/Response.d.ts

@@ -0,0 +1,36 @@
+/// <reference types="node" />
+/// <reference types="node" />
+import { Readable } from 'stream';
+import { IncomingHttpHeaders } from 'http';
+export declare type HttpClientResponseMeta = {
+    status: number;
+    statusCode: number;
+    headers: IncomingHttpHeaders;
+    size: number;
+    aborted: boolean;
+    rt: number;
+    keepAliveSocket: boolean;
+    requestUrls: string[];
+    /**
+     * https://eggjs.org/en/core/httpclient.html#timing-boolean
+     */
+    timing: {
+        contentDownload: number;
+        waiting: number;
+    };
+};
+export declare type ReadableWithMeta = Readable & {
+    status: number;
+    statusCode: number;
+    headers: IncomingHttpHeaders;
+};
+export declare type HttpClientResponse = {
+    opaque: unknown;
+    data: any;
+    status: number;
+    headers: IncomingHttpHeaders;
+    url: string;
+    redirected: boolean;
+    requestUrls: string[];
+    res: ReadableWithMeta | HttpClientResponseMeta;
+};

package/src/cjs/Response.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=Response.js.map

package/src/cjs/Response.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Response.js","sourceRoot":"","sources":["../Response.ts"],"names":[],"mappings":""}

package/src/cjs/index.d.ts

@@ -0,0 +1,7 @@
+import { RequestOptions, RequestURL } from './Request';
+export declare function request(url: RequestURL, options?: RequestOptions): Promise<import("./Response").HttpClientResponse>;
+export { HttpClient } from './HttpClient';
+declare const _default: {
+    request: typeof request;
+};
+export default _default;

package/src/cjs/index.js

@@ -0,0 +1,18 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HttpClient = exports.request = void 0;
+const HttpClient_1 = require("./HttpClient");
+let httpclient;
+async function request(url, options) {
+    if (!httpclient) {
+        httpclient = new HttpClient_1.HttpClient();
+    }
+    return await httpclient.request(url, options);
+}
+exports.request = request;
+var HttpClient_2 = require("./HttpClient");
+Object.defineProperty(exports, "HttpClient", { enumerable: true, get: function () { return HttpClient_2.HttpClient; } });
+exports.default = {
+    request,
+};
+//# sourceMappingURL=index.js.map

package/src/cjs/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":";;;AAAA,6CAA0C;AAG1C,IAAI,UAAsB,CAAC;AACpB,KAAK,UAAU,OAAO,CAAC,GAAe,EAAE,OAAwB;IACrE,IAAI,CAAC,UAAU,EAAE;QACf,UAAU,GAAG,IAAI,uBAAU,EAAE,CAAC;KAC/B;IACD,OAAO,MAAM,UAAU,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;AAChD,CAAC;AALD,0BAKC;AAED,2CAA0C;AAAjC,wGAAA,UAAU,OAAA;AAEnB,kBAAe;IACb,OAAO;CACR,CAAC"}

package/src/cjs/package.json

@@ -0,0 +1,3 @@
+{
+  "type": "commonjs"
+}

package/src/cjs/utils.d.ts

@@ -0,0 +1,3 @@
+import { FixJSONCtlChars } from './Request';
+export declare function parseJSON(data: string, fixJSONCtlChars?: FixJSONCtlChars): string;
+export declare function sleep(ms: number): Promise<void>;

package/src/cjs/utils.js

@@ -0,0 +1,56 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.sleep = exports.parseJSON = void 0;
+const JSONCtlCharsMap = {
+    '"': '\\"',
+    '\\': '\\\\',
+    '\b': '\\b',
+    '\f': '\\f',
+    '\n': '\\n',
+    '\r': '\\r',
+    '\t': '\\t', // \u0009
+};
+/* eslint no-control-regex: "off"*/
+const JSONCtlCharsRE = /[\u0000-\u001F\u005C]/g;
+function replaceOneChar(c) {
+    return JSONCtlCharsMap[c] || '\\u' + (c.charCodeAt(0) + 0x10000).toString(16).substring(1);
+}
+function replaceJSONCtlChars(value) {
+    return value.replace(JSONCtlCharsRE, replaceOneChar);
+}
+function parseJSON(data, fixJSONCtlChars) {
+    if (typeof fixJSONCtlChars === 'function') {
+        data = fixJSONCtlChars(data);
+    }
+    else if (fixJSONCtlChars) {
+        // https://github.com/node-modules/urllib/pull/77
+        // remote the control characters (U+0000 through U+001F)
+        data = replaceJSONCtlChars(data);
+    }
+    try {
+        data = JSON.parse(data);
+    }
+    catch (err) {
+        if (err.name === 'SyntaxError') {
+            err.name = 'JSONResponseFormatError';
+        }
+        if (data.length > 1024) {
+            // show 0~512 ... -512~end data
+            err.message += ' (data json format: ' +
+                JSON.stringify(data.slice(0, 512)) + ' ...skip... ' + JSON.stringify(data.slice(data.length - 512)) + ')';
+        }
+        else {
+            err.message += ' (data json format: ' + JSON.stringify(data) + ')';
+        }
+        throw err;
+    }
+    return data;
+}
+exports.parseJSON = parseJSON;
+function sleep(ms) {
+    return new Promise(resolve => {
+        setTimeout(resolve, ms);
+    });
+}
+exports.sleep = sleep;
+//# sourceMappingURL=utils.js.map

package/src/cjs/utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../utils.ts"],"names":[],"mappings":";;;AAEA,MAAM,eAAe,GAAG;IACtB,GAAG,EAAE,KAAK;IACV,IAAI,EAAE,MAAM;IACZ,IAAI,EAAE,KAAK;IACX,IAAI,EAAE,KAAK;IACX,IAAI,EAAE,KAAK;IACX,IAAI,EAAE,KAAK;IACX,IAAI,EAAE,KAAK,EAAE,SAAS;CACvB,CAAC;AACF,mCAAmC;AACnC,MAAM,cAAc,GAAG,wBAAwB,CAAC;AAEhD,SAAS,cAAc,CAAC,CAAS;IAC/B,OAAO,eAAe,CAAC,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;AAC7F,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAa;IACxC,OAAO,KAAK,CAAC,OAAO,CAAC,cAAc,EAAE,cAAc,CAAC,CAAC;AACvD,CAAC;AAED,SAAgB,SAAS,CAAC,IAAY,EAAE,eAAiC;IACvE,IAAI,OAAO,eAAe,KAAK,UAAU,EAAE;QACzC,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;KAC9B;SAAM,IAAI,eAAe,EAAE;QAC1B,iDAAiD;QACjD,wDAAwD;QACxD,IAAI,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAC;KAClC;IACD,IAAI;QACF,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;KACzB;IAAC,OAAO,GAAQ,EAAE;QACjB,IAAI,GAAG,CAAC,IAAI,KAAK,aAAa,EAAE;YAC9B,GAAG,CAAC,IAAI,GAAG,yBAAyB,CAAC;SACtC;QACD,IAAI,IAAI,CAAC,MAAM,GAAG,IAAI,EAAE;YACtB,+BAA+B;YAC/B,GAAG,CAAC,OAAO,IAAI,sBAAsB;gBACnC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,cAAc,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC;SAC7G;aAAM;YACL,GAAG,CAAC,OAAO,IAAI,sBAAsB,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC;SACpE;QACD,MAAM,GAAG,CAAC;KACX;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAxBD,8BAwBC;AAED,SAAgB,KAAK,CAAC,EAAU;IAC9B,OAAO,IAAI,OAAO,CAAO,OAAO,CAAC,EAAE;QACjC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IAC1B,CAAC,CAAC,CAAC;AACL,CAAC;AAJD,sBAIC"}

package/src/esm/HttpAgent.d.ts

@@ -0,0 +1,16 @@
+/// <reference types="node" />
+import { LookupFunction } from 'net';
+import { Agent } from 'undici';
+import { DispatchHandlers } from 'undici/types/dispatcher';
+import { BuildOptions } from 'undici/types/connector';
+export declare type CheckAddressFunction = (ip: string, family: number | string) => boolean;
+export declare type HttpAgentOptions = {
+    lookup?: LookupFunction;
+    checkAddress?: CheckAddressFunction;
+    connect?: BuildOptions;
+};
+export declare class HttpAgent extends Agent {
+    #private;
+    constructor(options: HttpAgentOptions);
+    dispatch(options: Agent.DispatchOptions, handler: DispatchHandlers): boolean;
+}

package/src/esm/HttpAgent.js

@@ -0,0 +1,58 @@
+var _HttpAgent_checkAddress;
+import { __classPrivateFieldGet, __classPrivateFieldSet } from "tslib";
+import dns from 'dns';
+import { isIP } from 'net';
+import { Agent, } from 'undici';
+class IllegalAddressError extends Error {
+    constructor(hostname, ip, family) {
+        const message = 'illegal address';
+        super(message);
+        this.name = this.constructor.name;
+        this.hostname = hostname;
+        this.ip = ip;
+        this.family = family;
+        Error.captureStackTrace(this, this.constructor);
+    }
+}
+export class HttpAgent extends Agent {
+    constructor(options) {
+        var _a;
+        /* eslint node/prefer-promises/dns: off*/
+        const _lookup = (_a = options.lookup) !== null && _a !== void 0 ? _a : dns.lookup;
+        const lookup = (hostname, dnsOptions, callback) => {
+            _lookup(hostname, dnsOptions, (err, address, family) => {
+                if (err)
+                    return callback(err, address, family);
+                if (options.checkAddress && !options.checkAddress(address, family)) {
+                    err = new IllegalAddressError(hostname, address, family);
+                }
+                callback(err, address, family);
+            });
+        };
+        super({
+            connect: { ...options.connect, lookup },
+        });
+        _HttpAgent_checkAddress.set(this, void 0);
+        __classPrivateFieldSet(this, _HttpAgent_checkAddress, options.checkAddress, "f");
+    }
+    dispatch(options, handler) {
+        if (__classPrivateFieldGet(this, _HttpAgent_checkAddress, "f") && options.origin) {
+            const originUrl = typeof options.origin === 'string' ? new URL(options.origin) : options.origin;
+            let hostname = originUrl.hostname;
+            // [2001:db8:2de::e13] => 2001:db8:2de::e13
+            if (hostname.startsWith('[') && hostname.endsWith(']')) {
+                hostname = hostname.substring(1, hostname.length - 1);
+            }
+            const family = isIP(hostname);
+            if (family === 4 || family === 6) {
+                // if request hostname is ip, custom lookup won't excute
+                if (!__classPrivateFieldGet(this, _HttpAgent_checkAddress, "f").call(this, hostname, family)) {
+                    throw new IllegalAddressError(hostname, hostname, family);
+                }
+            }
+        }
+        return super.dispatch(options, handler);
+    }
+}
+_HttpAgent_checkAddress = new WeakMap();
+//# sourceMappingURL=HttpAgent.js.map

package/src/esm/HttpAgent.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"HttpAgent.js","sourceRoot":"","sources":["../HttpAgent.ts"],"names":[],"mappings":";;AAAA,OAAO,GAAG,MAAM,KAAK,CAAC;AACtB,OAAO,EAAkB,IAAI,EAAE,MAAM,KAAK,CAAC;AAC3C,OAAO,EACL,KAAK,GACN,MAAM,QAAQ,CAAC;AAYhB,MAAM,mBAAoB,SAAQ,KAAK;IAKrC,YAAY,QAAgB,EAAE,EAAU,EAAE,MAAc;QACtD,MAAM,OAAO,GAAG,iBAAiB,CAAC;QAClC,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;QAClC,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,KAAK,CAAC,iBAAiB,CAAC,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IAClD,CAAC;CACF;AAED,MAAM,OAAO,SAAU,SAAQ,KAAK;IAGlC,YAAY,OAAyB;;QACnC,yCAAyC;QACzC,MAAM,OAAO,GAAG,MAAA,OAAO,CAAC,MAAM,mCAAI,GAAG,CAAC,MAAM,CAAC;QAC7C,MAAM,MAAM,GAAmB,CAAC,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,EAAE;YAChE,OAAO,CAAC,QAAQ,EAAE,UAAU,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE;gBACrD,IAAI,GAAG;oBAAE,OAAO,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;gBAC/C,IAAI,OAAO,CAAC,YAAY,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE;oBAClE,GAAG,GAAG,IAAI,mBAAmB,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;iBAC1D;gBACD,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;YACjC,CAAC,CAAC,CAAC;QACL,CAAC,CAAC;QACF,KAAK,CAAC;YACJ,OAAO,EAAE,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE;SACxC,CAAC,CAAC;QAhBL,0CAAqC;QAiBnC,uBAAA,IAAI,2BAAiB,OAAO,CAAC,YAAY,MAAA,CAAC;IAC5C,CAAC;IAED,QAAQ,CAAC,OAA8B,EAAE,OAAyB;QAChE,IAAI,uBAAA,IAAI,+BAAc,IAAI,OAAO,CAAC,MAAM,EAAE;YACxC,MAAM,SAAS,GAAG,OAAO,OAAO,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;YAChG,IAAI,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAC;YAClC,2CAA2C;YAC3C,IAAI,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;gBACtD,QAAQ,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;aACvD;YACD,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC9B,IAAI,MAAM,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC,EAAE;gBAChC,wDAAwD;gBACxD,IAAI,CAAC,uBAAA,IAAI,+BAAc,MAAlB,IAAI,EAAe,QAAQ,EAAE,MAAM,CAAC,EAAE;oBACzC,MAAM,IAAI,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;iBAC3D;aACF;SACF;QACD,OAAO,KAAK,CAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC1C,CAAC;CACF"}

package/src/esm/HttpClient.d.ts

@@ -0,0 +1,39 @@
+/// <reference types="node" />
+/// <reference types="node" />
+/// <reference types="node" />
+import { EventEmitter } from 'events';
+import { LookupFunction } from 'net';
+import { CheckAddressFunction } from './HttpAgent';
+import { RequestURL, RequestOptions } from './Request';
+import { HttpClientResponse } from './Response';
+export declare type ClientOptions = {
+    defaultArgs?: RequestOptions;
+    /**
+     * Custom DNS lookup function, default is `dns.lookup`.
+     */
+    lookup?: LookupFunction;
+    /**
+      * check request address to protect from SSRF and similar attacks.
+      * It receive two arguments(ip and family) and should return true or false to identified the address is legal or not.
+      * It rely on lookup and have the same version requirement.
+      */
+    checkAddress?: CheckAddressFunction;
+    connect?: {
+        key?: string | Buffer;
+        /**
+        * A string or Buffer containing the certificate key of the client in PEM format.
+        * Notes: This is necessary only if using the client certificate authentication
+        */
+        cert?: string | Buffer;
+        /**
+        * If true, the server certificate is verified against the list of supplied CAs.
+        * An 'error' event is emitted if verification fails.Default: true.
+        */
+        rejectUnauthorized?: boolean;
+    };
+};
+export declare class HttpClient extends EventEmitter {
+    #private;
+    constructor(clientOptions?: ClientOptions);
+    request(url: RequestURL, options?: RequestOptions): Promise<HttpClientResponse>;
+}