urllib 3.22.1 → 3.22.3

package/README.md

@@ -68,7 +68,7 @@ console.log('status: %s, body size: %d, headers: %j', res.status, data.length, r
   - **keepAliveTimeout** `number | null` - Default is `4000`, 4 seconds - The timeout after which a socket without active requests will time out. Monitors time between activity on a connected socket. This value may be overridden by *keep-alive* hints from the server. See [MDN: HTTP - Headers - Keep-Alive directives](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Keep-Alive#directives) for more details.
   - ***auth*** String - `username:password` used in HTTP Basic Authorization.
   - ***digestAuth*** String - `username:password` used in HTTP [Digest Authorization](https://en.wikipedia.org/wiki/Digest_access_authentication).
-  - ***followRedirect*** Boolean - follow HTTP 3xx responses as redirects. defaults to false.
+  - ***followRedirect*** Boolean - follow HTTP 3xx responses as redirects. defaults to true.
   - ***maxRedirects*** Number - The maximum number of redirects to follow, defaults to 10.
   - ***formatRedirectUrl*** Function - Format the redirect url by your self. Default is `url.resolve(from, to)`.
   - ***beforeRequest*** Function - Before request hook, you can change every thing here.

package/dist/commonjs/HttpClient.js

@@ -16,6 +16,7 @@ const node_path_1 = require("node:path");
 const node_fs_1 = require("node:fs");
 const node_url_1 = require("node:url");
 const node_perf_hooks_1 = require("node:perf_hooks");
+const node_querystring_1 = __importDefault(require("node:querystring"));
 const undici_1 = require("undici");
 const symbols_js_1 = __importDefault(require("undici/lib/core/symbols.js"));
 const formdata_node_1 = require("formdata-node");
@@ -67,7 +68,7 @@ class BlobFromStream {
         return 'Blob';
     }
 }
-exports.HEADER_USER_AGENT = (0, default_user_agent_1.default)('node-urllib', '3.22.1');
+exports.HEADER_USER_AGENT = (0, default_user_agent_1.default)('node-urllib', '3.22.3');
 function getFileName(stream) {
     const filePath = stream.path;
     if (filePath) {
@@ -412,20 +413,16 @@ class HttpClient extends node_events_1.EventEmitter {
                     || (0, utils_js_1.isReadable)(args.data);
                 if (isGETOrHEAD) {
                     if (!isStringOrBufferOrReadable) {
+                        let query;
                         if (args.nestedQuerystring) {
-                            const querystring = qs_1.default.stringify(args.data);
-                            // reset the requestUrl
-                            const href = requestUrl.href;
-                            requestUrl = new URL(href + (href.includes('?') ? '&' : '?') + querystring);
+                            query = qs_1.default.stringify(args.data);
                         }
                         else {
-                            for (const field in args.data) {
-                                const fieldValue = args.data[field];
-                                if (fieldValue === undefined)
-                                    continue;
-                                requestUrl.searchParams.append(field, fieldValue);
-                            }
+                            query = node_querystring_1.default.stringify(args.data);
                         }
+                        // reset the requestUrl
+                        const href = requestUrl.href;
+                        requestUrl = new URL(href + (href.includes('?') ? '&' : '?') + query);
                     }
                 }
                 else {

package/dist/commonjs/diagnosticsChannel.js

@@ -56,6 +56,21 @@ node_net_1.Socket.prototype.destroy = function (err) {
     }
     return this[kDestroy](err);
 };
+function getRequestOpaque(request, kHandler) {
+    if (!kHandler)
+        return;
+    const handler = request[kHandler];
+    // maxRedirects = 0 will get [Symbol(handler)]: RequestHandler {
+    // responseHeaders: null,
+    // opaque: {
+    //   [Symbol(request id)]: 1,
+    //   [Symbol(request start time)]: 465.0712921619415,
+    //   [Symbol(enable request timing or not)]: true,
+    //   [Symbol(request timing)]: [Object],
+    //   [Symbol(request original opaque)]: undefined
+    // }
+    return handler?.opts?.opaque ?? handler?.opaque;
+}
 function initDiagnosticsChannel() {
     // makre sure init global DiagnosticsChannel once
     if (initedDiagnosticsChannel)
@@ -75,7 +90,7 @@ function initDiagnosticsChannel() {
                 }
             }
         }
-        const opaque = request[kHandler]?.opts?.opaque;
+        const opaque = getRequestOpaque(request, kHandler);
         // ignore non HttpClient Request
         if (!opaque || !opaque[symbols_js_1.default.kRequestId])
             return;
@@ -129,9 +144,7 @@ function initDiagnosticsChannel() {
     // This message is published right before the first byte of the request is written to the socket.
     subscribe('undici:client:sendHeaders', (message, name) => {
         const { request, socket } = message;
-        if (!kHandler)
-            return;
-        const opaque = request[kHandler]?.opts?.opaque;
+        const opaque = getRequestOpaque(request, kHandler);
         if (!opaque || !opaque[symbols_js_1.default.kRequestId])
             return;
         socket[symbols_js_1.default.kHandledRequests]++;
@@ -150,9 +163,7 @@ function initDiagnosticsChannel() {
     });
     subscribe('undici:request:bodySent', (message, name) => {
         const { request } = message;
-        if (!kHandler)
-            return;
-        const opaque = request[kHandler]?.opts?.opaque;
+        const opaque = getRequestOpaque(request, kHandler);
         if (!opaque || !opaque[symbols_js_1.default.kRequestId])
             return;
         debug('[%s] Request#%d send body', name, opaque[symbols_js_1.default.kRequestId]);
@@ -163,9 +174,7 @@ function initDiagnosticsChannel() {
     // This message is published after the response headers have been received, i.e. the response has been completed.
     subscribe('undici:request:headers', (message, name) => {
         const { request, response } = message;
-        if (!kHandler)
-            return;
-        const opaque = request[kHandler]?.opts?.opaque;
+        const opaque = getRequestOpaque(request, kHandler);
         if (!opaque || !opaque[symbols_js_1.default.kRequestId])
             return;
         // get socket from opaque
@@ -179,9 +188,7 @@ function initDiagnosticsChannel() {
     // This message is published after the response body and trailers have been received, i.e. the response has been completed.
     subscribe('undici:request:trailers', (message, name) => {
         const { request } = message;
-        if (!kHandler)
-            return;
-        const opaque = request[kHandler]?.opts?.opaque;
+        const opaque = getRequestOpaque(request, kHandler);
         if (!opaque || !opaque[symbols_js_1.default.kRequestId])
             return;
         debug('[%s] Request#%d get response body and trailers', name, opaque[symbols_js_1.default.kRequestId]);

package/dist/esm/HttpClient.js

@@ -10,6 +10,7 @@ import { basename } from 'node:path';
 import { createReadStream } from 'node:fs';
 import { format as urlFormat } from 'node:url';
 import { performance } from 'node:perf_hooks';
+import querystring from 'node:querystring';
 import { FormData as FormDataNative, request as undiciRequest, Agent, getGlobalDispatcher, } from 'undici';
 import undiciSymbols from 'undici/lib/core/symbols.js';
 import { FormData as FormDataNode } from 'formdata-node';
@@ -61,7 +62,7 @@ class BlobFromStream {
         return 'Blob';
     }
 }
-export const HEADER_USER_AGENT = createUserAgent('node-urllib', '3.22.1');
+export const HEADER_USER_AGENT = createUserAgent('node-urllib', '3.22.3');
 function getFileName(stream) {
     const filePath = stream.path;
     if (filePath) {
@@ -406,20 +407,16 @@ export class HttpClient extends EventEmitter {
                     || isReadable(args.data);
                 if (isGETOrHEAD) {
                     if (!isStringOrBufferOrReadable) {
+                        let query;
                         if (args.nestedQuerystring) {
-                            const querystring = qs.stringify(args.data);
-                            // reset the requestUrl
-                            const href = requestUrl.href;
-                            requestUrl = new URL(href + (href.includes('?') ? '&' : '?') + querystring);
+                            query = qs.stringify(args.data);
                         }
                         else {
-                            for (const field in args.data) {
-                                const fieldValue = args.data[field];
-                                if (fieldValue === undefined)
-                                    continue;
-                                requestUrl.searchParams.append(field, fieldValue);
-                            }
+                            query = querystring.stringify(args.data);
                         }
+                        // reset the requestUrl
+                        const href = requestUrl.href;
+                        requestUrl = new URL(href + (href.includes('?') ? '&' : '?') + query);
                     }
                 }
                 else {

package/dist/esm/diagnosticsChannel.js

@@ -50,6 +50,21 @@ Socket.prototype.destroy = function (err) {
     }
     return this[kDestroy](err);
 };
+function getRequestOpaque(request, kHandler) {
+    if (!kHandler)
+        return;
+    const handler = request[kHandler];
+    // maxRedirects = 0 will get [Symbol(handler)]: RequestHandler {
+    // responseHeaders: null,
+    // opaque: {
+    //   [Symbol(request id)]: 1,
+    //   [Symbol(request start time)]: 465.0712921619415,
+    //   [Symbol(enable request timing or not)]: true,
+    //   [Symbol(request timing)]: [Object],
+    //   [Symbol(request original opaque)]: undefined
+    // }
+    return handler?.opts?.opaque ?? handler?.opaque;
+}
 export function initDiagnosticsChannel() {
     // makre sure init global DiagnosticsChannel once
     if (initedDiagnosticsChannel)
@@ -69,7 +84,7 @@ export function initDiagnosticsChannel() {
                 }
             }
         }
-        const opaque = request[kHandler]?.opts?.opaque;
+        const opaque = getRequestOpaque(request, kHandler);
         // ignore non HttpClient Request
         if (!opaque || !opaque[symbols.kRequestId])
             return;
@@ -123,9 +138,7 @@ export function initDiagnosticsChannel() {
     // This message is published right before the first byte of the request is written to the socket.
     subscribe('undici:client:sendHeaders', (message, name) => {
         const { request, socket } = message;
-        if (!kHandler)
-            return;
-        const opaque = request[kHandler]?.opts?.opaque;
+        const opaque = getRequestOpaque(request, kHandler);
         if (!opaque || !opaque[symbols.kRequestId])
             return;
         socket[symbols.kHandledRequests]++;
@@ -144,9 +157,7 @@ export function initDiagnosticsChannel() {
     });
     subscribe('undici:request:bodySent', (message, name) => {
         const { request } = message;
-        if (!kHandler)
-            return;
-        const opaque = request[kHandler]?.opts?.opaque;
+        const opaque = getRequestOpaque(request, kHandler);
         if (!opaque || !opaque[symbols.kRequestId])
             return;
         debug('[%s] Request#%d send body', name, opaque[symbols.kRequestId]);
@@ -157,9 +168,7 @@ export function initDiagnosticsChannel() {
     // This message is published after the response headers have been received, i.e. the response has been completed.
     subscribe('undici:request:headers', (message, name) => {
         const { request, response } = message;
-        if (!kHandler)
-            return;
-        const opaque = request[kHandler]?.opts?.opaque;
+        const opaque = getRequestOpaque(request, kHandler);
         if (!opaque || !opaque[symbols.kRequestId])
             return;
         // get socket from opaque
@@ -173,9 +182,7 @@ export function initDiagnosticsChannel() {
     // This message is published after the response body and trailers have been received, i.e. the response has been completed.
     subscribe('undici:request:trailers', (message, name) => {
         const { request } = message;
-        if (!kHandler)
-            return;
-        const opaque = request[kHandler]?.opts?.opaque;
+        const opaque = getRequestOpaque(request, kHandler);
         if (!opaque || !opaque[symbols.kRequestId])
             return;
         debug('[%s] Request#%d get response body and trailers', name, opaque[symbols.kRequestId]);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "urllib",
-  "version": "3.22.1",
+  "version": "3.22.3",
   "publishConfig": {
     "tag": "latest"
   },
@@ -55,7 +55,7 @@
     "pump": "^3.0.0",
     "qs": "^6.11.2",
     "type-fest": "^4.3.1",
-    "undici": "^5.22.1",
+    "undici": "^5.28.2",
     "ylru": "^1.3.2"
   },
   "devDependencies": {

package/src/HttpClient.ts

@@ -16,6 +16,7 @@ import { basename } from 'node:path';
 import { createReadStream } from 'node:fs';
 import { format as urlFormat } from 'node:url';
 import { performance } from 'node:perf_hooks';
+import querystring from 'node:querystring';
 import {
   FormData as FormDataNative,
   request as undiciRequest,
@@ -503,18 +504,15 @@ export class HttpClient extends EventEmitter {
           || isReadable(args.data);
         if (isGETOrHEAD) {
           if (!isStringOrBufferOrReadable) {
+            let query;
             if (args.nestedQuerystring) {
-              const querystring = qs.stringify(args.data);
-              // reset the requestUrl
-              const href = requestUrl.href;
-              requestUrl = new URL(href + (href.includes('?') ? '&' : '?') + querystring);
+              query = qs.stringify(args.data);
             } else {
-              for (const field in args.data) {
-                const fieldValue = args.data[field];
-                if (fieldValue === undefined) continue;
-                requestUrl.searchParams.append(field, fieldValue);
-              }
+              query = querystring.stringify(args.data);
             }
+            // reset the requestUrl
+            const href = requestUrl.href;
+            requestUrl = new URL(href + (href.includes('?') ? '&' : '?') + query);
           }
         } else {
           if (isStringOrBufferOrReadable) {

package/src/diagnosticsChannel.ts

@@ -54,6 +54,21 @@ Socket.prototype.destroy = function(err?: any) {
   return this[kDestroy](err);
 };
 
+function getRequestOpaque(request: DiagnosticsChannel.Request, kHandler?: symbol) {
+  if (!kHandler) return;
+  const handler = request[kHandler];
+  // maxRedirects = 0 will get [Symbol(handler)]: RequestHandler {
+  // responseHeaders: null,
+  // opaque: {
+  //   [Symbol(request id)]: 1,
+  //   [Symbol(request start time)]: 465.0712921619415,
+  //   [Symbol(enable request timing or not)]: true,
+  //   [Symbol(request timing)]: [Object],
+  //   [Symbol(request original opaque)]: undefined
+  // }
+  return handler?.opts?.opaque ?? handler?.opaque;
+}
+
 export function initDiagnosticsChannel() {
   // makre sure init global DiagnosticsChannel once
   if (initedDiagnosticsChannel) return;
@@ -73,7 +88,7 @@ export function initDiagnosticsChannel() {
         }
       }
     }
-    const opaque = request[kHandler]?.opts?.opaque;
+    const opaque = getRequestOpaque(request, kHandler);
     // ignore non HttpClient Request
     if (!opaque || !opaque[symbols.kRequestId]) return;
     debug('[%s] Request#%d %s %s, path: %s, headers: %o',
@@ -131,8 +146,7 @@ export function initDiagnosticsChannel() {
   // This message is published right before the first byte of the request is written to the socket.
   subscribe('undici:client:sendHeaders', (message, name) => {
     const { request, socket } = message as DiagnosticsChannel.ClientSendHeadersMessage;
-    if (!kHandler) return;
-    const opaque = request[kHandler]?.opts?.opaque;
+    const opaque = getRequestOpaque(request, kHandler);
     if (!opaque || !opaque[symbols.kRequestId]) return;
 
     socket[symbols.kHandledRequests]++;
@@ -154,8 +168,7 @@ export function initDiagnosticsChannel() {
 
   subscribe('undici:request:bodySent', (message, name) => {
     const { request } = message as DiagnosticsChannel.RequestBodySentMessage;
-    if (!kHandler) return;
-    const opaque = request[kHandler]?.opts?.opaque;
+    const opaque = getRequestOpaque(request, kHandler);
     if (!opaque || !opaque[symbols.kRequestId]) return;
 
     debug('[%s] Request#%d send body', name, opaque[symbols.kRequestId]);
@@ -166,8 +179,7 @@ export function initDiagnosticsChannel() {
   // This message is published after the response headers have been received, i.e. the response has been completed.
   subscribe('undici:request:headers', (message, name) => {
     const { request, response } = message as DiagnosticsChannel.RequestHeadersMessage;
-    if (!kHandler) return;
-    const opaque = request[kHandler]?.opts?.opaque;
+    const opaque = getRequestOpaque(request, kHandler);
     if (!opaque || !opaque[symbols.kRequestId]) return;
 
     // get socket from opaque
@@ -184,8 +196,7 @@ export function initDiagnosticsChannel() {
   // This message is published after the response body and trailers have been received, i.e. the response has been completed.
   subscribe('undici:request:trailers', (message, name) => {
     const { request } = message as DiagnosticsChannel.RequestTrailersMessage;
-    if (!kHandler) return;
-    const opaque = request[kHandler]?.opts?.opaque;
+    const opaque = getRequestOpaque(request, kHandler);
     if (!opaque || !opaque[symbols.kRequestId]) return;
 
     debug('[%s] Request#%d get response body and trailers', name, opaque[symbols.kRequestId]);