urllib 3.17.1 → 3.18.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "urllib",
-  "version": "3.17.1",
+  "version": "3.18.0",
   "publishConfig": {
     "tag": "latest"
   },
@@ -68,6 +68,7 @@
     "formstream": "^1.1.1",
     "mime-types": "^2.1.35",
     "pump": "^3.0.0",
+    "qs": "^6.11.2",
     "undici": "^5.22.1",
     "ylru": "^1.3.2"
   },
@@ -77,6 +78,7 @@
     "@types/mime-types": "^2.1.1",
     "@types/node": "^20.2.1",
     "@types/pump": "^1.1.1",
+    "@types/qs": "^6.9.7",
     "@types/selfsigned": "^2.0.1",
     "@types/tar-stream": "^2.2.2",
     "@vitest/coverage-v8": "^0.32.0",

package/src/HttpAgent.ts

@@ -37,8 +37,11 @@ export class HttpAgent extends Agent {
     /* eslint node/prefer-promises/dns: off*/
     const _lookup = options.lookup ?? dns.lookup;
     const lookup: LookupFunction = (hostname, dnsOptions, callback) => {
-      _lookup(hostname, dnsOptions, (err, address, family) => {
-        if (err) return callback(err, address, family);
+      _lookup(hostname, dnsOptions, (err, ...args: any[]) => {
+        // address will be array on Node.js >= 20
+        const address = args[0];
+        const family = args[1];
+        if (err) return (callback as any)(err, address, family);
         if (options.checkAddress) {
           // dnsOptions.all set to default on Node.js >= 20, dns.lookup will return address array object
           if (typeof address === 'string') {
@@ -55,7 +58,7 @@ export class HttpAgent extends Agent {
             }
           }
         }
-        callback(err, address, family);
+        (callback as any)(err, address, family);
       });
     };
     super({

package/src/HttpClient.ts

@@ -26,6 +26,7 @@ import { FormData as FormDataNode } from 'formdata-node';
 import { FormDataEncoder } from 'form-data-encoder';
 import createUserAgent from 'default-user-agent';
 import mime from 'mime-types';
+import qs from 'qs';
 import pump from 'pump';
 // Compatible with old style formstream
 import FormStream from 'formstream';
@@ -58,6 +59,8 @@ function noop() {
 }
 
 const debug = debuglog('urllib:HttpClient');
+// Node.js 14 or 16
+const isNode14Or16 = /v1[46]\./.test(process.version);
 
 export type ClientOptions = {
   defaultArgs?: RequestOptions;
@@ -85,7 +88,7 @@ export type ClientOptions = {
     rejectUnauthorized?: boolean;
 
     /**
-     * sockePath string | null (optional) - Default: null - An IPC endpoint, either Unix domain socket or Windows named pipe
+     * socketPath string | null (optional) - Default: null - An IPC endpoint, either Unix domain socket or Windows named pipe
      */
     socketPath?: string | null;
   },
@@ -242,14 +245,14 @@ export class HttpClient extends EventEmitter {
       // the response body and trailers have been received
       contentDownload: 0,
     };
-    const orginalOpaque = args.opaque;
+    const originalOpaque = args.opaque;
     // using opaque to diagnostics channel, binding request and socket
     const internalOpaque = {
       [symbols.kRequestId]: requestId,
       [symbols.kRequestStartTime]: requestStartTime,
       [symbols.kEnableRequestTiming]: !!args.timing,
       [symbols.kRequestTiming]: timing,
-      [symbols.kRequestOrginalOpaque]: orginalOpaque,
+      [symbols.kRequestOriginalOpaque]: originalOpaque,
     };
     const reqMeta = {
       requestId,
@@ -450,10 +453,17 @@ export class HttpClient extends EventEmitter {
           || isReadable(args.data);
         if (isGETOrHEAD) {
           if (!isStringOrBufferOrReadable) {
-            for (const field in args.data) {
-              const fieldValue = args.data[field];
-              if (fieldValue === undefined) continue;
-              requestUrl.searchParams.append(field, fieldValue);
+            if (args.nestedQuerystring) {
+              const querystring = qs.stringify(args.data);
+              // reset the requestUrl
+              const href = requestUrl.href;
+              requestUrl = new URL(href + (href.includes('?') ? '&' : '?') + querystring);
+            } else {
+              for (const field in args.data) {
+                const fieldValue = args.data[field];
+                if (fieldValue === undefined) continue;
+                requestUrl.searchParams.append(field, fieldValue);
+              }
             }
           }
         } else {
@@ -470,7 +480,11 @@ export class HttpClient extends EventEmitter {
               }
             } else {
               headers['content-type'] = 'application/x-www-form-urlencoded;charset=UTF-8';
-              requestOptions.body = new URLSearchParams(args.data).toString();
+              if (args.nestedQuerystring) {
+                requestOptions.body = qs.stringify(args.data);
+              } else {
+                requestOptions.body = new URLSearchParams(args.data).toString();
+              }
             }
           }
         }
@@ -543,6 +557,9 @@ export class HttpClient extends EventEmitter {
           res = Object.assign(response.body, res);
         }
       } else if (args.writeStream) {
+        if (isNode14Or16 && args.writeStream.destroyed) {
+          throw new Error('writeStream is destroyed');
+        }
         if (args.compressed === true && isCompressedContent) {
           const decoder = contentEncoding === 'gzip' ? createGunzip() : createBrotliDecompress();
           await pipelinePromise(response.body, decoder, args.writeStream);
@@ -577,7 +594,7 @@ export class HttpClient extends EventEmitter {
       this.#updateSocketInfo(socketInfo, internalOpaque);
 
       const clientResponse: HttpClientResponse = {
-        opaque: orginalOpaque,
+        opaque: originalOpaque,
         data,
         status: res.status,
         statusCode: res.status,
@@ -632,7 +649,7 @@ export class HttpClient extends EventEmitter {
           return await this.#requestInternal(url, options, requestContext);
         }
       }
-      err.opaque = orginalOpaque;
+      err.opaque = originalOpaque;
       err.status = res.status;
       err.headers = res.headers;
       err.res = res;

package/src/Request.ts

@@ -46,6 +46,11 @@ export type RequestOptions = {
    * Default is 'buffer'.
    */
   dataType?: 'text' | 'html' | 'json' | 'buffer' | 'stream';
+  /**
+   * urllib default use URLSearchParams to stringify form data which don't support nested object,
+   * will use qs instead of URLSearchParams to support nested object by set this option to true.
+   */
+  nestedQuerystring?: boolean;
   /**
    * @deprecated
    * Only for d.ts keep compatible with urllib@2, don't use it anymore.

package/src/cjs/HttpAgent.js

@@ -27,7 +27,10 @@ class HttpAgent extends undici_1.Agent {
         /* eslint node/prefer-promises/dns: off*/
         const _lookup = options.lookup ?? node_dns_1.default.lookup;
         const lookup = (hostname, dnsOptions, callback) => {
-            _lookup(hostname, dnsOptions, (err, address, family) => {
+            _lookup(hostname, dnsOptions, (err, ...args) => {
+                // address will be array on Node.js >= 20
+                const address = args[0];
+                const family = args[1];
                 if (err)
                     return callback(err, address, family);
                 if (options.checkAddress) {

package/src/cjs/HttpClient.d.ts

@@ -31,7 +31,7 @@ export type ClientOptions = {
         */
         rejectUnauthorized?: boolean;
         /**
-         * sockePath string | null (optional) - Default: null - An IPC endpoint, either Unix domain socket or Windows named pipe
+         * socketPath string | null (optional) - Default: null - An IPC endpoint, either Unix domain socket or Windows named pipe
          */
         socketPath?: string | null;
     };

package/src/cjs/HttpClient.js

@@ -21,6 +21,7 @@ const formdata_node_1 = require("formdata-node");
 const form_data_encoder_1 = require("form-data-encoder");
 const default_user_agent_1 = __importDefault(require("default-user-agent"));
 const mime_types_1 = __importDefault(require("mime-types"));
+const qs_1 = __importDefault(require("qs"));
 const pump_1 = __importDefault(require("pump"));
 // Compatible with old style formstream
 const formstream_1 = __importDefault(require("formstream"));
@@ -44,6 +45,8 @@ function noop() {
     // noop
 }
 const debug = (0, node_util_1.debuglog)('urllib:HttpClient');
+// Node.js 14 or 16
+const isNode14Or16 = /v1[46]\./.test(process.version);
 // https://github.com/octet-stream/form-data
 class BlobFromStream {
     #stream;
@@ -70,7 +73,7 @@ class HttpClientRequestTimeoutError extends Error {
         Error.captureStackTrace(this, this.constructor);
     }
 }
-exports.HEADER_USER_AGENT = (0, default_user_agent_1.default)('node-urllib', '3.17.1');
+exports.HEADER_USER_AGENT = (0, default_user_agent_1.default)('node-urllib', '3.18.0');
 function getFileName(stream) {
     const filePath = stream.path;
     if (filePath) {
@@ -165,14 +168,14 @@ class HttpClient extends node_events_1.EventEmitter {
             // the response body and trailers have been received
             contentDownload: 0,
         };
-        const orginalOpaque = args.opaque;
+        const originalOpaque = args.opaque;
         // using opaque to diagnostics channel, binding request and socket
         const internalOpaque = {
             [symbols_1.default.kRequestId]: requestId,
             [symbols_1.default.kRequestStartTime]: requestStartTime,
             [symbols_1.default.kEnableRequestTiming]: !!args.timing,
             [symbols_1.default.kRequestTiming]: timing,
-            [symbols_1.default.kRequestOrginalOpaque]: orginalOpaque,
+            [symbols_1.default.kRequestOriginalOpaque]: originalOpaque,
         };
         const reqMeta = {
             requestId,
@@ -379,11 +382,19 @@ class HttpClient extends node_events_1.EventEmitter {
                     || (0, utils_1.isReadable)(args.data);
                 if (isGETOrHEAD) {
                     if (!isStringOrBufferOrReadable) {
-                        for (const field in args.data) {
-                            const fieldValue = args.data[field];
-                            if (fieldValue === undefined)
-                                continue;
-                            requestUrl.searchParams.append(field, fieldValue);
+                        if (args.nestedQuerystring) {
+                            const querystring = qs_1.default.stringify(args.data);
+                            // reset the requestUrl
+                            const href = requestUrl.href;
+                            requestUrl = new URL(href + (href.includes('?') ? '&' : '?') + querystring);
+                        }
+                        else {
+                            for (const field in args.data) {
+                                const fieldValue = args.data[field];
+                                if (fieldValue === undefined)
+                                    continue;
+                                requestUrl.searchParams.append(field, fieldValue);
+                            }
                         }
                     }
                 }
@@ -403,7 +414,12 @@ class HttpClient extends node_events_1.EventEmitter {
                         }
                         else {
                             headers['content-type'] = 'application/x-www-form-urlencoded;charset=UTF-8';
-                            requestOptions.body = new URLSearchParams(args.data).toString();
+                            if (args.nestedQuerystring) {
+                                requestOptions.body = qs_1.default.stringify(args.data);
+                            }
+                            else {
+                                requestOptions.body = new URLSearchParams(args.data).toString();
+                            }
                         }
                     }
                 }
@@ -472,6 +488,9 @@ class HttpClient extends node_events_1.EventEmitter {
                 }
             }
             else if (args.writeStream) {
+                if (isNode14Or16 && args.writeStream.destroyed) {
+                    throw new Error('writeStream is destroyed');
+                }
                 if (args.compressed === true && isCompressedContent) {
                     const decoder = contentEncoding === 'gzip' ? (0, node_zlib_1.createGunzip)() : (0, node_zlib_1.createBrotliDecompress)();
                     await pipelinePromise(response.body, decoder, args.writeStream);
@@ -510,7 +529,7 @@ class HttpClient extends node_events_1.EventEmitter {
             // get real socket info from internalOpaque
             this.#updateSocketInfo(socketInfo, internalOpaque);
             const clientResponse = {
-                opaque: orginalOpaque,
+                opaque: originalOpaque,
                 data,
                 status: res.status,
                 statusCode: res.status,
@@ -565,7 +584,7 @@ class HttpClient extends node_events_1.EventEmitter {
                     return await this.#requestInternal(url, options, requestContext);
                 }
             }
-            err.opaque = orginalOpaque;
+            err.opaque = originalOpaque;
             err.status = res.status;
             err.headers = res.headers;
             err.res = res;

package/src/cjs/Request.d.ts

@@ -44,6 +44,11 @@ export type RequestOptions = {
      * Default is 'buffer'.
      */
     dataType?: 'text' | 'html' | 'json' | 'buffer' | 'stream';
+    /**
+     * urllib default use URLSearchParams to stringify form data which don't support nested object,
+     * will use qs instead of URLSearchParams to support nested object by set this option to true.
+     */
+    nestedQuerystring?: boolean;
     /**
      * @deprecated
      * Only for d.ts keep compatible with urllib@2, don't use it anymore.

package/src/cjs/symbols.d.ts

@@ -12,6 +12,6 @@ declare const _default: {
     kRequestStartTime: symbol;
     kEnableRequestTiming: symbol;
     kRequestTiming: symbol;
-    kRequestOrginalOpaque: symbol;
+    kRequestOriginalOpaque: symbol;
 };
 export default _default;

package/src/cjs/symbols.js

@@ -14,5 +14,5 @@ exports.default = {
     kRequestStartTime: Symbol('request start time'),
     kEnableRequestTiming: Symbol('enable request timing or not'),
     kRequestTiming: Symbol('request timing'),
-    kRequestOrginalOpaque: Symbol('request orginal opaque'),
+    kRequestOriginalOpaque: Symbol('request original opaque'),
 };

package/src/esm/HttpAgent.js

@@ -21,7 +21,10 @@ export class HttpAgent extends Agent {
         /* eslint node/prefer-promises/dns: off*/
         const _lookup = options.lookup ?? dns.lookup;
         const lookup = (hostname, dnsOptions, callback) => {
-            _lookup(hostname, dnsOptions, (err, address, family) => {
+            _lookup(hostname, dnsOptions, (err, ...args) => {
+                // address will be array on Node.js >= 20
+                const address = args[0];
+                const family = args[1];
                 if (err)
                     return callback(err, address, family);
                 if (options.checkAddress) {

package/src/esm/HttpClient.d.ts

@@ -31,7 +31,7 @@ export type ClientOptions = {
         */
         rejectUnauthorized?: boolean;
         /**
-         * sockePath string | null (optional) - Default: null - An IPC endpoint, either Unix domain socket or Windows named pipe
+         * socketPath string | null (optional) - Default: null - An IPC endpoint, either Unix domain socket or Windows named pipe
          */
         socketPath?: string | null;
     };

package/src/esm/HttpClient.js

@@ -15,6 +15,7 @@ import { FormData as FormDataNode } from 'formdata-node';
 import { FormDataEncoder } from 'form-data-encoder';
 import createUserAgent from 'default-user-agent';
 import mime from 'mime-types';
+import qs from 'qs';
 import pump from 'pump';
 // Compatible with old style formstream
 import FormStream from 'formstream';
@@ -38,6 +39,8 @@ function noop() {
     // noop
 }
 const debug = debuglog('urllib:HttpClient');
+// Node.js 14 or 16
+const isNode14Or16 = /v1[46]\./.test(process.version);
 // https://github.com/octet-stream/form-data
 class BlobFromStream {
     #stream;
@@ -64,7 +67,7 @@ class HttpClientRequestTimeoutError extends Error {
         Error.captureStackTrace(this, this.constructor);
     }
 }
-export const HEADER_USER_AGENT = createUserAgent('node-urllib', '3.17.1');
+export const HEADER_USER_AGENT = createUserAgent('node-urllib', '3.18.0');
 function getFileName(stream) {
     const filePath = stream.path;
     if (filePath) {
@@ -159,14 +162,14 @@ export class HttpClient extends EventEmitter {
             // the response body and trailers have been received
             contentDownload: 0,
         };
-        const orginalOpaque = args.opaque;
+        const originalOpaque = args.opaque;
         // using opaque to diagnostics channel, binding request and socket
         const internalOpaque = {
             [symbols.kRequestId]: requestId,
             [symbols.kRequestStartTime]: requestStartTime,
             [symbols.kEnableRequestTiming]: !!args.timing,
             [symbols.kRequestTiming]: timing,
-            [symbols.kRequestOrginalOpaque]: orginalOpaque,
+            [symbols.kRequestOriginalOpaque]: originalOpaque,
         };
         const reqMeta = {
             requestId,
@@ -373,11 +376,19 @@ export class HttpClient extends EventEmitter {
                     || isReadable(args.data);
                 if (isGETOrHEAD) {
                     if (!isStringOrBufferOrReadable) {
-                        for (const field in args.data) {
-                            const fieldValue = args.data[field];
-                            if (fieldValue === undefined)
-                                continue;
-                            requestUrl.searchParams.append(field, fieldValue);
+                        if (args.nestedQuerystring) {
+                            const querystring = qs.stringify(args.data);
+                            // reset the requestUrl
+                            const href = requestUrl.href;
+                            requestUrl = new URL(href + (href.includes('?') ? '&' : '?') + querystring);
+                        }
+                        else {
+                            for (const field in args.data) {
+                                const fieldValue = args.data[field];
+                                if (fieldValue === undefined)
+                                    continue;
+                                requestUrl.searchParams.append(field, fieldValue);
+                            }
                         }
                     }
                 }
@@ -397,7 +408,12 @@ export class HttpClient extends EventEmitter {
                         }
                         else {
                             headers['content-type'] = 'application/x-www-form-urlencoded;charset=UTF-8';
-                            requestOptions.body = new URLSearchParams(args.data).toString();
+                            if (args.nestedQuerystring) {
+                                requestOptions.body = qs.stringify(args.data);
+                            }
+                            else {
+                                requestOptions.body = new URLSearchParams(args.data).toString();
+                            }
                         }
                     }
                 }
@@ -466,6 +482,9 @@ export class HttpClient extends EventEmitter {
                 }
             }
             else if (args.writeStream) {
+                if (isNode14Or16 && args.writeStream.destroyed) {
+                    throw new Error('writeStream is destroyed');
+                }
                 if (args.compressed === true && isCompressedContent) {
                     const decoder = contentEncoding === 'gzip' ? createGunzip() : createBrotliDecompress();
                     await pipelinePromise(response.body, decoder, args.writeStream);
@@ -504,7 +523,7 @@ export class HttpClient extends EventEmitter {
             // get real socket info from internalOpaque
             this.#updateSocketInfo(socketInfo, internalOpaque);
             const clientResponse = {
-                opaque: orginalOpaque,
+                opaque: originalOpaque,
                 data,
                 status: res.status,
                 statusCode: res.status,
@@ -559,7 +578,7 @@ export class HttpClient extends EventEmitter {
                     return await this.#requestInternal(url, options, requestContext);
                 }
             }
-            err.opaque = orginalOpaque;
+            err.opaque = originalOpaque;
             err.status = res.status;
             err.headers = res.headers;
             err.res = res;

package/src/esm/Request.d.ts

@@ -44,6 +44,11 @@ export type RequestOptions = {
      * Default is 'buffer'.
      */
     dataType?: 'text' | 'html' | 'json' | 'buffer' | 'stream';
+    /**
+     * urllib default use URLSearchParams to stringify form data which don't support nested object,
+     * will use qs instead of URLSearchParams to support nested object by set this option to true.
+     */
+    nestedQuerystring?: boolean;
     /**
      * @deprecated
      * Only for d.ts keep compatible with urllib@2, don't use it anymore.

package/src/esm/symbols.d.ts

@@ -12,6 +12,6 @@ declare const _default: {
     kRequestStartTime: symbol;
     kEnableRequestTiming: symbol;
     kRequestTiming: symbol;
-    kRequestOrginalOpaque: symbol;
+    kRequestOriginalOpaque: symbol;
 };
 export default _default;

package/src/esm/symbols.js

@@ -12,5 +12,5 @@ export default {
     kRequestStartTime: Symbol('request start time'),
     kEnableRequestTiming: Symbol('enable request timing or not'),
     kRequestTiming: Symbol('request timing'),
-    kRequestOrginalOpaque: Symbol('request orginal opaque'),
+    kRequestOriginalOpaque: Symbol('request original opaque'),
 };

package/src/symbols.ts

@@ -12,5 +12,5 @@ export default {
   kRequestStartTime: Symbol('request start time'),
   kEnableRequestTiming: Symbol('enable request timing or not'),
   kRequestTiming: Symbol('request timing'),
-  kRequestOrginalOpaque: Symbol('request orginal opaque'),
+  kRequestOriginalOpaque: Symbol('request original opaque'),
 };