urllib 3.0.0-alpha.0 → 3.0.1

package/src/esm/HttpClient.js

@@ -1,14 +1,43 @@
-var _BlobFromStream_stream, _BlobFromStream_type;
+var _a, _b, _c;
+var _BlobFromStream_stream, _BlobFromStream_type, _HttpClient_instances, _HttpClient_defaultArgs, _HttpClient_dispatcher, _HttpClient_requestInternal;
 import { __classPrivateFieldGet, __classPrivateFieldSet } from "tslib";
 import { EventEmitter } from 'events';
 import { debuglog } from 'util';
-import { Readable, isReadable } from 'stream';
+import { createGunzip, createBrotliDecompress, gunzipSync, brotliDecompressSync, } from 'zlib';
 import { Blob } from 'buffer';
-import { createReadStream } from 'fs';
+import { Readable, pipeline } from 'stream';
+import stream from 'stream';
 import { basename } from 'path';
-import { fetch, Headers, FormData, } from 'undici';
+import { createReadStream } from 'fs';
+import { performance } from 'perf_hooks';
+import { FormData as FormDataNative, request as undiciRequest, } from 'undici';
+import { FormData as FormDataNode } from 'formdata-node';
+import { FormDataEncoder } from 'form-data-encoder';
 import createUserAgent from 'default-user-agent';
 import mime from 'mime-types';
+import pump from 'pump';
+import { HttpAgent } from './HttpAgent.js';
+import { parseJSON, sleep } from './utils.js';
+const FormData = FormDataNative !== null && FormDataNative !== void 0 ? FormDataNative : FormDataNode;
+// impl isReadable on Node.js 14
+const isReadable = (_a = stream.isReadable) !== null && _a !== void 0 ? _a : function isReadable(stream) {
+    return stream && typeof stream.read === 'function';
+};
+// impl promise pipeline on Node.js 14
+const pipelinePromise = (_c = (_b = stream.promises) === null || _b === void 0 ? void 0 : _b.pipeline) !== null && _c !== void 0 ? _c : function pipeline(...args) {
+    return new Promise((resolve, reject) => {
+        pump(...args, (err) => {
+            if (err)
+                return reject(err);
+            resolve();
+        });
+    });
+};
+function noop() {
+    // noop
+}
+const MAX_REQURE_ID_VALUE = Math.pow(2, 31) - 10;
+let globalRequestId = 0;
 const debug = debuglog('urllib');
 // https://github.com/octet-stream/form-data
 class BlobFromStream {
@@ -36,7 +65,7 @@ class HttpClientRequestTimeoutError extends Error {
         Error.captureStackTrace(this, this.constructor);
     }
 }
-const HEADER_USER_AGENT = createUserAgent('node-urllib', '3.0.0');
+export const HEADER_USER_AGENT = createUserAgent('node-urllib', '3.0.0');
 function getFileName(stream) {
     const filePath = stream.path;
     if (filePath) {
@@ -44,223 +73,390 @@ function getFileName(stream) {
     }
     return '';
 }
+function defaultIsRetry(response) {
+    return response.status >= 500;
+}
+function performanceTime(startTime) {
+    return Math.floor((performance.now() - startTime) * 1000) / 1000;
+}
 export class HttpClient extends EventEmitter {
     constructor(clientOptions) {
         super();
-        this.defaultArgs = clientOptions === null || clientOptions === void 0 ? void 0 : clientOptions.defaultArgs;
+        _HttpClient_instances.add(this);
+        _HttpClient_defaultArgs.set(this, void 0);
+        _HttpClient_dispatcher.set(this, void 0);
+        __classPrivateFieldSet(this, _HttpClient_defaultArgs, clientOptions === null || clientOptions === void 0 ? void 0 : clientOptions.defaultArgs, "f");
+        if ((clientOptions === null || clientOptions === void 0 ? void 0 : clientOptions.lookup) || (clientOptions === null || clientOptions === void 0 ? void 0 : clientOptions.checkAddress) || (clientOptions === null || clientOptions === void 0 ? void 0 : clientOptions.connect)) {
+            __classPrivateFieldSet(this, _HttpClient_dispatcher, new HttpAgent({
+                lookup: clientOptions.lookup,
+                checkAddress: clientOptions.checkAddress,
+                connect: clientOptions.connect,
+            }), "f");
+        }
     }
     async request(url, options) {
-        var _a, _b, _c, _d;
-        const requestUrl = typeof url === 'string' ? new URL(url) : url;
-        const args = {
-            ...this.defaultArgs,
-            ...options,
-            emitter: this,
-        };
-        const requestStartTime = Date.now();
-        // keep urllib createCallbackResponse style
-        const resHeaders = {};
-        const res = {
-            status: -1,
-            statusCode: -1,
-            statusMessage: '',
-            headers: resHeaders,
-            size: 0,
-            aborted: false,
-            rt: 0,
-            keepAliveSocket: true,
-            requestUrls: [url.toString()],
-            timing: {
-                contentDownload: 0,
-            },
-            // remoteAddress: remoteAddress,
-            // remotePort: remotePort,
-            // socketHandledRequests: socketHandledRequests,
-            // socketHandledResponses: socketHandledResponses,
+        return await __classPrivateFieldGet(this, _HttpClient_instances, "m", _HttpClient_requestInternal).call(this, url, options);
+    }
+}
+_HttpClient_defaultArgs = new WeakMap(), _HttpClient_dispatcher = new WeakMap(), _HttpClient_instances = new WeakSet(), _HttpClient_requestInternal = async function _HttpClient_requestInternal(url, options, requestContext) {
+    var _a, _b, _c, _d, _e, _f;
+    if (globalRequestId >= MAX_REQURE_ID_VALUE) {
+        globalRequestId = 0;
+    }
+    const requestId = ++globalRequestId;
+    const requestUrl = typeof url === 'string' ? new URL(url) : url;
+    const args = {
+        retry: 0,
+        ...__classPrivateFieldGet(this, _HttpClient_defaultArgs, "f"),
+        ...options,
+    };
+    requestContext = {
+        retries: 0,
+        ...requestContext,
+    };
+    const requestStartTime = performance.now();
+    const reqMeta = {
+        requestId,
+        url: requestUrl.href,
+        args,
+        ctx: args.ctx,
+    };
+    // keep urllib createCallbackResponse style
+    const resHeaders = {};
+    const res = {
+        status: -1,
+        statusCode: -1,
+        headers: resHeaders,
+        size: 0,
+        aborted: false,
+        rt: 0,
+        keepAliveSocket: true,
+        requestUrls: [],
+        timing: {
+            waiting: 0,
+            contentDownload: 0,
+        },
+    };
+    let headersTimeout = 5000;
+    let bodyTimeout = 5000;
+    if (args.timeout) {
+        if (Array.isArray(args.timeout)) {
+            headersTimeout = (_a = args.timeout[0]) !== null && _a !== void 0 ? _a : headersTimeout;
+            bodyTimeout = (_b = args.timeout[1]) !== null && _b !== void 0 ? _b : bodyTimeout;
+        }
+        else {
+            headersTimeout = bodyTimeout = args.timeout;
+        }
+    }
+    const method = ((_c = args.method) !== null && _c !== void 0 ? _c : 'GET').toUpperCase();
+    const headers = {};
+    if (args.headers) {
+        // convert headers to lower-case
+        for (const name in args.headers) {
+            headers[name.toLowerCase()] = args.headers[name];
+        }
+    }
+    // hidden user-agent
+    const hiddenUserAgent = 'user-agent' in headers && !headers['user-agent'];
+    if (hiddenUserAgent) {
+        delete headers['user-agent'];
+    }
+    else if (!headers['user-agent']) {
+        // need to set user-agent
+        headers['user-agent'] = HEADER_USER_AGENT;
+    }
+    // Alias to dataType = 'stream'
+    if (args.streaming || args.customResponse) {
+        args.dataType = 'stream';
+    }
+    if (args.dataType === 'json' && !headers.accept) {
+        headers.accept = 'application/json';
+    }
+    // gzip alias to compressed
+    if (args.gzip && args.compressed !== false) {
+        args.compressed = true;
+    }
+    if (args.compressed && !headers['accept-encoding']) {
+        headers['accept-encoding'] = 'gzip, br';
+    }
+    if (requestContext.retries > 0) {
+        headers['x-urllib-retry'] = `${requestContext.retries}/${args.retry}`;
+    }
+    if (args.auth && !headers.authorization) {
+        headers.authorization = `Basic ${Buffer.from(args.auth).toString('base64')}`;
+    }
+    let opaque = args.opaque;
+    try {
+        const requestOptions = {
+            method,
+            keepalive: true,
+            maxRedirections: (_d = args.maxRedirects) !== null && _d !== void 0 ? _d : 10,
+            headersTimeout,
+            bodyTimeout,
+            opaque,
+            dispatcher: __classPrivateFieldGet(this, _HttpClient_dispatcher, "f"),
         };
-        let requestTimeout = 5000;
-        if (args.timeout) {
-            if (Array.isArray(args.timeout)) {
-                requestTimeout = (_a = args.timeout[args.timeout.length - 1]) !== null && _a !== void 0 ? _a : requestTimeout;
-            }
-            else {
-                requestTimeout = args.timeout;
-            }
+        if (args.followRedirect === false) {
+            requestOptions.maxRedirections = 0;
+        }
+        const isGETOrHEAD = requestOptions.method === 'GET' || requestOptions.method === 'HEAD';
+        // alias to args.content
+        if (args.stream && !args.content) {
+            args.content = args.stream;
         }
-        const requestTimeoutController = new AbortController();
-        const requestTimerId = setTimeout(() => requestTimeoutController.abort(), requestTimeout);
-        const method = ((_b = args.method) !== null && _b !== void 0 ? _b : 'GET').toUpperCase();
-        try {
-            const headers = new Headers((_c = args.headers) !== null && _c !== void 0 ? _c : {});
-            // don't set user-agent
-            const disableUserAgent = args.headers &&
-                (args.headers['User-Agent'] === null || args.headers['user-agent'] === null);
-            if (!disableUserAgent && !headers.has('user-agent')) {
-                // need to set user-agent
-                headers.set('user-agent', HEADER_USER_AGENT);
+        if (args.files) {
+            if (isGETOrHEAD) {
+                requestOptions.method = 'POST';
             }
-            if (args.dataType === 'json' && !headers.has('accept')) {
-                headers.set('accept', 'application/json');
+            const formData = new FormData();
+            const uploadFiles = [];
+            if (Array.isArray(args.files)) {
+                for (const [index, file] of args.files.entries()) {
+                    const field = index === 0 ? 'file' : `file${index}`;
+                    uploadFiles.push([field, file]);
+                }
             }
-            const requestOptions = {
-                method,
-                keepalive: true,
-                signal: requestTimeoutController.signal,
-            };
-            if (args.followRedirect === false) {
-                requestOptions.redirect = 'manual';
+            else if (args.files instanceof Readable || isReadable(args.files)) {
+                uploadFiles.push(['file', args.files]);
+            }
+            else if (typeof args.files === 'string' || Buffer.isBuffer(args.files)) {
+                uploadFiles.push(['file', args.files]);
             }
-            const isGETOrHEAD = requestOptions.method === 'GET' || requestOptions.method === 'HEAD';
-            if (args.files) {
-                if (isGETOrHEAD) {
-                    requestOptions.method = 'POST';
+            else if (typeof args.files === 'object') {
+                for (const field in args.files) {
+                    uploadFiles.push([field, args.files[field]]);
                 }
-                const formData = new FormData();
-                const uploadFiles = [];
-                if (Array.isArray(args.files)) {
-                    for (const [index, file] of args.files.entries()) {
-                        const field = index === 0 ? 'file' : `file${index}`;
-                        uploadFiles.push([field, file]);
-                    }
+            }
+            // set normal fields first
+            if (args.data) {
+                for (const field in args.data) {
+                    formData.append(field, args.data[field]);
                 }
-                else if (args.files instanceof Readable || isReadable(args.files)) {
-                    uploadFiles.push(['file', args.files]);
+            }
+            for (const [index, [field, file]] of uploadFiles.entries()) {
+                if (typeof file === 'string') {
+                    // FIXME: support non-ascii filename
+                    // const fileName = encodeURIComponent(basename(file));
+                    // formData.append(field, await fileFromPath(file, `utf-8''${fileName}`, { type: mime.lookup(fileName) || '' }));
+                    const fileName = basename(file);
+                    const fileReadable = createReadStream(file);
+                    formData.append(field, new BlobFromStream(fileReadable, mime.lookup(fileName) || ''), fileName);
                 }
-                else if (typeof args.files === 'string' || Buffer.isBuffer(args.files)) {
-                    uploadFiles.push(['file', args.files]);
+                else if (Buffer.isBuffer(file)) {
+                    formData.append(field, new Blob([file]), `bufferfile${index}`);
                 }
-                else if (typeof args.files === 'object') {
-                    for (const field in args.files) {
-                        uploadFiles.push([field, args.files[field]]);
-                    }
+                else if (file instanceof Readable || isReadable(file)) {
+                    const fileName = getFileName(file) || `streamfile${index}`;
+                    formData.append(field, new BlobFromStream(file, mime.lookup(fileName) || ''), fileName);
                 }
-                // set normal fields first
-                if (args.data) {
-                    for (const field in args.data) {
-                        formData.append(field, args.data[field]);
-                    }
+            }
+            if (FormDataNative) {
+                requestOptions.body = formData;
+            }
+            else {
+                // Node.js 14 does not support spec-compliant FormData
+                // https://github.com/octet-stream/form-data#usage
+                const encoder = new FormDataEncoder(formData);
+                Object.assign(headers, encoder.headers);
+                // fix "Content-Length":"NaN"
+                delete headers['Content-Length'];
+                requestOptions.body = Readable.from(encoder);
+            }
+        }
+        else if (args.content) {
+            if (!isGETOrHEAD) {
+                // handle content
+                requestOptions.body = args.content;
+                if (args.contentType) {
+                    headers['content-type'] = args.contentType;
                 }
-                for (const [index, [field, file]] of uploadFiles.entries()) {
-                    if (typeof file === 'string') {
-                        // FIXME: support non-ascii filename
-                        // const fileName = encodeURIComponent(basename(file));
-                        // formData.append(field, await fileFromPath(file, `utf-8''${fileName}`, { type: mime.lookup(fileName) || '' }));
-                        const fileName = basename(file);
-                        const fileReader = createReadStream(file);
-                        formData.append(field, new BlobFromStream(fileReader, mime.lookup(fileName) || ''), fileName);
-                    }
-                    else if (Buffer.isBuffer(file)) {
-                        formData.append(field, new Blob([file]), `bufferfile${index}`);
-                    }
-                    else if (file instanceof Readable || isReadable(file)) {
-                        const fileName = getFileName(file) || `streamfile${index}`;
-                        formData.append(field, new BlobFromStream(file, mime.lookup(fileName) || ''), fileName);
-                    }
+                if (typeof args.content === 'string' && !headers['content-type']) {
+                    headers['content-type'] = 'text/plain;charset=UTF-8';
                 }
-                requestOptions.body = formData;
             }
-            else if (args.content) {
-                if (!isGETOrHEAD) {
-                    if (isReadable(args.content)) {
-                        // disable keepalive
-                        requestOptions.keepalive = false;
-                    }
-                    // handle content
-                    requestOptions.body = args.content;
-                    if (args.contentType) {
-                        headers.set('content-type', args.contentType);
+        }
+        else if (args.data) {
+            const isStringOrBufferOrReadable = typeof args.data === 'string'
+                || Buffer.isBuffer(args.data)
+                || isReadable(args.data);
+            if (isGETOrHEAD) {
+                if (!isStringOrBufferOrReadable) {
+                    for (const field in args.data) {
+                        requestUrl.searchParams.append(field, args.data[field]);
                     }
                 }
             }
-            else if (args.data) {
-                const isStringOrBufferOrReadable = typeof args.data === 'string'
-                    || Buffer.isBuffer(args.data)
-                    || isReadable(args.data);
-                if (isGETOrHEAD) {
-                    if (!isStringOrBufferOrReadable) {
-                        for (const field in args.data) {
-                            requestUrl.searchParams.append(field, args.data[field]);
-                        }
-                    }
+            else {
+                if (isStringOrBufferOrReadable) {
+                    requestOptions.body = args.data;
                 }
                 else {
-                    if (isStringOrBufferOrReadable) {
-                        if (isReadable(args.data)) {
-                            // disable keepalive
-                            requestOptions.keepalive = false;
+                    if (args.contentType === 'json'
+                        || args.contentType === 'application/json'
+                        || ((_e = headers['content-type']) === null || _e === void 0 ? void 0 : _e.startsWith('application/json'))) {
+                        requestOptions.body = JSON.stringify(args.data);
+                        if (!headers['content-type']) {
+                            headers['content-type'] = 'application/json';
                         }
-                        requestOptions.body = args.data;
                     }
                     else {
-                        if (args.contentType === 'json'
-                            || args.contentType === 'application/json'
-                            || ((_d = headers.get('content-type')) === null || _d === void 0 ? void 0 : _d.startsWith('application/json'))) {
-                            requestOptions.body = JSON.stringify(args.data);
-                            if (!headers.has('content-type')) {
-                                headers.set('content-type', 'application/json');
-                            }
-                        }
-                        else {
-                            requestOptions.body = new URLSearchParams(args.data);
-                        }
+                        headers['content-type'] = 'application/x-www-form-urlencoded;charset=UTF-8';
+                        requestOptions.body = new URLSearchParams(args.data).toString();
                     }
                 }
             }
-            debug('%s %s, headers: %j, timeout: %s', requestOptions.method, url, headers, requestTimeout);
-            requestOptions.headers = headers;
-            const response = await fetch(requestUrl, requestOptions);
-            for (const [name, value] of response.headers) {
-                res.headers[name] = value;
+        }
+        debug('Request#%d %s %s, headers: %j, headersTimeout: %s, bodyTimeout: %s', requestId, requestOptions.method, requestUrl.href, headers, headersTimeout, bodyTimeout);
+        requestOptions.headers = headers;
+        if (this.listenerCount('request') > 0) {
+            this.emit('request', reqMeta);
+        }
+        const response = await undiciRequest(requestUrl, requestOptions);
+        opaque = response.opaque;
+        if (args.timing) {
+            res.timing.waiting = performanceTime(requestStartTime);
+        }
+        const context = response.context;
+        let lastUrl = '';
+        if (context === null || context === void 0 ? void 0 : context.history) {
+            for (const urlObject of context === null || context === void 0 ? void 0 : context.history) {
+                res.requestUrls.push(urlObject.href);
+                lastUrl = urlObject.href;
+            }
+        }
+        else {
+            res.requestUrls.push(requestUrl.href);
+            lastUrl = requestUrl.href;
+        }
+        const contentEncoding = response.headers['content-encoding'];
+        const isCompressedContent = contentEncoding === 'gzip' || contentEncoding === 'br';
+        res.headers = response.headers;
+        res.status = res.statusCode = response.statusCode;
+        if (res.headers['content-length']) {
+            res.size = parseInt(res.headers['content-length']);
+        }
+        let data = null;
+        let responseBodyStream;
+        if (args.dataType === 'stream') {
+            // streaming mode will disable retry
+            args.retry = 0;
+            const meta = {
+                status: res.status,
+                statusCode: res.statusCode,
+                headers: res.headers,
+            };
+            if (isCompressedContent) {
+                // gzip or br
+                const decoder = contentEncoding === 'gzip' ? createGunzip() : createBrotliDecompress();
+                responseBodyStream = Object.assign(pipeline(response.body, decoder, noop), meta);
+            }
+            else {
+                responseBodyStream = Object.assign(response.body, meta);
             }
-            res.status = res.statusCode = response.status;
-            res.statusMessage = response.statusText;
-            if (response.redirected) {
-                res.requestUrls.push(response.url);
+        }
+        else if (args.writeStream) {
+            // streaming mode will disable retry
+            args.retry = 0;
+            if (isCompressedContent) {
+                const decoder = contentEncoding === 'gzip' ? createGunzip() : createBrotliDecompress();
+                await pipelinePromise(response.body, decoder, args.writeStream);
             }
-            if (res.headers['content-length']) {
-                res.size = parseInt(res.headers['content-length']);
+            else {
+                await pipelinePromise(response.body, args.writeStream);
             }
-            let data;
-            if (args.streaming || args.dataType === 'stream') {
-                data = response.body;
+        }
+        else {
+            // buffer
+            data = Buffer.from(await response.body.arrayBuffer());
+            if (isCompressedContent) {
+                try {
+                    data = contentEncoding === 'gzip' ? gunzipSync(data) : brotliDecompressSync(data);
+                }
+                catch (err) {
+                    if (err.name === 'Error') {
+                        err.name = 'UnzipError';
+                    }
+                    throw err;
+                }
             }
-            else if (args.dataType === 'text') {
-                data = await response.text();
+            if (args.dataType === 'text') {
+                data = data.toString();
             }
             else if (args.dataType === 'json') {
-                if (requestOptions.method === 'HEAD') {
-                    data = {};
+                if (data.length === 0) {
+                    data = null;
                 }
                 else {
-                    data = await response.json();
+                    data = parseJSON(data.toString(), args.fixJSONCtlChars);
                 }
             }
-            else {
-                // buffer
-                data = Buffer.from(await response.arrayBuffer());
+        }
+        res.rt = performanceTime(requestStartTime);
+        if (args.timing) {
+            res.timing.contentDownload = res.rt;
+        }
+        const clientResponse = {
+            opaque,
+            data,
+            status: res.status,
+            headers: res.headers,
+            url: lastUrl,
+            redirected: res.requestUrls.length > 1,
+            requestUrls: res.requestUrls,
+            res: responseBodyStream !== null && responseBodyStream !== void 0 ? responseBodyStream : res,
+        };
+        if (args.retry > 0 && requestContext.retries < args.retry) {
+            const isRetry = (_f = args.isRetry) !== null && _f !== void 0 ? _f : defaultIsRetry;
+            if (isRetry(clientResponse)) {
+                if (args.retryDelay) {
+                    await sleep(args.retryDelay);
+                }
+                requestContext.retries++;
+                return await __classPrivateFieldGet(this, _HttpClient_instances, "m", _HttpClient_requestInternal).call(this, url, options, requestContext);
             }
-            res.rt = res.timing.contentDownload = Date.now() - requestStartTime;
-            return {
-                status: res.status,
-                data,
-                headers: res.headers,
-                url: response.url,
-                redirected: response.redirected,
+        }
+        if (this.listenerCount('response') > 0) {
+            this.emit('response', {
+                requestId,
+                error: null,
+                ctx: args.ctx,
+                req: reqMeta,
                 res,
-            };
+            });
         }
-        catch (e) {
-            let err = e;
-            if (requestTimeoutController.signal.aborted) {
-                err = new HttpClientRequestTimeoutError(requestTimeout, { cause: e });
-            }
-            err.res = res;
-            // console.error(err);
-            throw err;
+        return clientResponse;
+    }
+    catch (e) {
+        debug('Request#%d throw error: %s', requestId, e);
+        let err = e;
+        if (err.name === 'HeadersTimeoutError') {
+            err = new HttpClientRequestTimeoutError(headersTimeout, { cause: e });
         }
-        finally {
-            clearTimeout(requestTimerId);
+        else if (err.name === 'BodyTimeoutError') {
+            err = new HttpClientRequestTimeoutError(bodyTimeout, { cause: e });
         }
+        err.opaque = opaque;
+        err.status = res.status;
+        err.headers = res.headers;
+        err.res = res;
+        // make sure requestUrls not empty
+        if (res.requestUrls.length === 0) {
+            res.requestUrls.push(requestUrl.href);
+        }
+        res.rt = performanceTime(requestStartTime);
+        if (args.timing) {
+            res.timing.contentDownload = res.rt;
+        }
+        if (this.listenerCount('response') > 0) {
+            this.emit('response', {
+                requestId,
+                error: err,
+                ctx: args.ctx,
+                req: reqMeta,
+                res,
+            });
+        }
+        throw err;
     }
-}
+};
 //# sourceMappingURL=HttpClient.js.map