urllib 3.21.0 → 3.22.0

package/dist/commonjs/HttpClient.d.ts

@@ -42,6 +42,11 @@ export type ClientOptions = {
     };
 };
 export declare const HEADER_USER_AGENT: string;
+export type RequestContext = {
+    retries: number;
+    socketErrorRetries: number;
+    requestStartTime?: number;
+};
 export type RequestDiagnosticsMessage = {
     request: RequestMeta;
 };
@@ -50,11 +55,26 @@ export type ResponseDiagnosticsMessage = {
     response: RawResponseWithMeta;
     error?: Error;
 };
+export interface PoolStat {
+    /** Number of open socket connections in this pool. */
+    connected: number;
+    /** Number of open socket connections in this pool that do not have an active request. */
+    free: number;
+    /** Number of pending requests across all clients in this pool. */
+    pending: number;
+    /** Number of queued requests across all clients in this pool. */
+    queued: number;
+    /** Number of currently active requests across all clients in this pool. */
+    running: number;
+    /** Number of active, pending, or queued requests across all clients in this pool. */
+    size: number;
+}
 export declare class HttpClient extends EventEmitter {
     #private;
     constructor(clientOptions?: ClientOptions);
     getDispatcher(): Dispatcher;
     setDispatcher(dispatcher: Dispatcher): void;
+    getDispatcherPoolStats(): Record<string, PoolStat>;
     request<T = any>(url: RequestURL, options?: RequestOptions): Promise<HttpClientResponse<T>>;
     curl<T = any>(url: RequestURL, options?: RequestOptions): Promise<HttpClientResponse<T>>;
 }

package/dist/commonjs/HttpClient.js

@@ -17,6 +17,7 @@ const node_fs_1 = require("node:fs");
 const node_url_1 = require("node:url");
 const node_perf_hooks_1 = require("node:perf_hooks");
 const undici_1 = require("undici");
+const symbols_js_1 = __importDefault(require("undici/lib/core/symbols.js"));
 const formdata_node_1 = require("formdata-node");
 const form_data_encoder_1 = require("form-data-encoder");
 const default_user_agent_1 = __importDefault(require("default-user-agent"));
@@ -27,7 +28,7 @@ const pump_1 = __importDefault(require("pump"));
 const formstream_1 = __importDefault(require("formstream"));
 const HttpAgent_js_1 = require("./HttpAgent.js");
 const utils_js_1 = require("./utils.js");
-const symbols_js_1 = __importDefault(require("./symbols.js"));
+const symbols_js_2 = __importDefault(require("./symbols.js"));
 const diagnosticsChannel_js_1 = require("./diagnosticsChannel.js");
 const HttpClientError_js_1 = require("./HttpClientError.js");
 const PROTO_RE = /^https?:\/\//i;
@@ -66,7 +67,7 @@ class BlobFromStream {
         return 'Blob';
     }
 }
-exports.HEADER_USER_AGENT = (0, default_user_agent_1.default)('node-urllib', '3.21.0');
+exports.HEADER_USER_AGENT = (0, default_user_agent_1.default)('node-urllib', '3.22.0');
 function getFileName(stream) {
     const filePath = stream.path;
     if (filePath) {
@@ -107,10 +108,31 @@ class HttpClient extends node_events_1.EventEmitter {
     setDispatcher(dispatcher) {
         this.#dispatcher = dispatcher;
     }
+    getDispatcherPoolStats() {
+        const agent = this.getDispatcher();
+        // origin => Pool Instance
+        const clients = agent[symbols_js_1.default.kClients];
+        const poolStatsMap = {};
+        for (const [key, ref] of clients) {
+            const pool = ref.deref();
+            const stats = pool?.stats;
+            if (!stats)
+                continue;
+            poolStatsMap[key] = {
+                connected: stats.connected,
+                free: stats.free,
+                pending: stats.pending,
+                queued: stats.queued,
+                running: stats.running,
+                size: stats.size,
+            };
+        }
+        return poolStatsMap;
+    }
     async request(url, options) {
         return await this.#requestInternal(url, options);
     }
-    // alias to request, keep compatible with urlib@2 HttpClient.curl
+    // alias to request, keep compatible with urllib@2 HttpClient.curl
     async curl(url, options) {
         return await this.request(url, options);
     }
@@ -176,11 +198,11 @@ class HttpClient extends node_events_1.EventEmitter {
         const originalOpaque = args.opaque;
         // using opaque to diagnostics channel, binding request and socket
         const internalOpaque = {
-            [symbols_js_1.default.kRequestId]: requestId,
-            [symbols_js_1.default.kRequestStartTime]: requestStartTime,
-            [symbols_js_1.default.kEnableRequestTiming]: !!args.timing,
-            [symbols_js_1.default.kRequestTiming]: timing,
-            [symbols_js_1.default.kRequestOriginalOpaque]: originalOpaque,
+            [symbols_js_2.default.kRequestId]: requestId,
+            [symbols_js_2.default.kRequestStartTime]: requestStartTime,
+            [symbols_js_2.default.kEnableRequestTiming]: !!args.timing,
+            [symbols_js_2.default.kRequestTiming]: timing,
+            [symbols_js_2.default.kRequestOriginalOpaque]: originalOpaque,
         };
         const reqMeta = {
             requestId,
@@ -628,14 +650,14 @@ class HttpClient extends node_events_1.EventEmitter {
         }
     }
     #updateSocketInfo(socketInfo, internalOpaque, err) {
-        const socket = internalOpaque[symbols_js_1.default.kRequestSocket] ?? err?.[symbols_js_1.default.kErrorSocket];
+        const socket = internalOpaque[symbols_js_2.default.kRequestSocket] ?? err?.[symbols_js_2.default.kErrorSocket];
         if (socket) {
-            socketInfo.id = socket[symbols_js_1.default.kSocketId];
-            socketInfo.handledRequests = socket[symbols_js_1.default.kHandledRequests];
-            socketInfo.handledResponses = socket[symbols_js_1.default.kHandledResponses];
-            if (socket[symbols_js_1.default.kSocketLocalAddress]) {
-                socketInfo.localAddress = socket[symbols_js_1.default.kSocketLocalAddress];
-                socketInfo.localPort = socket[symbols_js_1.default.kSocketLocalPort];
+            socketInfo.id = socket[symbols_js_2.default.kSocketId];
+            socketInfo.handledRequests = socket[symbols_js_2.default.kHandledRequests];
+            socketInfo.handledResponses = socket[symbols_js_2.default.kHandledResponses];
+            if (socket[symbols_js_2.default.kSocketLocalAddress]) {
+                socketInfo.localAddress = socket[symbols_js_2.default.kSocketLocalAddress];
+                socketInfo.localPort = socket[symbols_js_2.default.kSocketLocalPort];
             }
             if (socket.remoteAddress) {
                 socketInfo.remoteAddress = socket.remoteAddress;
@@ -644,22 +666,22 @@ class HttpClient extends node_events_1.EventEmitter {
             }
             socketInfo.bytesRead = socket.bytesRead;
             socketInfo.bytesWritten = socket.bytesWritten;
-            if (socket[symbols_js_1.default.kSocketConnectErrorTime]) {
-                socketInfo.connectErrorTime = socket[symbols_js_1.default.kSocketConnectErrorTime];
+            if (socket[symbols_js_2.default.kSocketConnectErrorTime]) {
+                socketInfo.connectErrorTime = socket[symbols_js_2.default.kSocketConnectErrorTime];
                 if (Array.isArray(socket.autoSelectFamilyAttemptedAddresses)) {
                     socketInfo.attemptedRemoteAddresses = socket.autoSelectFamilyAttemptedAddresses;
                 }
-                socketInfo.connectProtocol = socket[symbols_js_1.default.kSocketConnectProtocol];
-                socketInfo.connectHost = socket[symbols_js_1.default.kSocketConnectHost];
-                socketInfo.connectPort = socket[symbols_js_1.default.kSocketConnectPort];
+                socketInfo.connectProtocol = socket[symbols_js_2.default.kSocketConnectProtocol];
+                socketInfo.connectHost = socket[symbols_js_2.default.kSocketConnectHost];
+                socketInfo.connectPort = socket[symbols_js_2.default.kSocketConnectPort];
             }
-            if (socket[symbols_js_1.default.kSocketConnectedTime]) {
-                socketInfo.connectedTime = socket[symbols_js_1.default.kSocketConnectedTime];
+            if (socket[symbols_js_2.default.kSocketConnectedTime]) {
+                socketInfo.connectedTime = socket[symbols_js_2.default.kSocketConnectedTime];
             }
-            if (socket[symbols_js_1.default.kSocketRequestEndTime]) {
-                socketInfo.lastRequestEndTime = socket[symbols_js_1.default.kSocketRequestEndTime];
+            if (socket[symbols_js_2.default.kSocketRequestEndTime]) {
+                socketInfo.lastRequestEndTime = socket[symbols_js_2.default.kSocketRequestEndTime];
             }
-            socket[symbols_js_1.default.kSocketRequestEndTime] = new Date();
+            socket[symbols_js_2.default.kSocketRequestEndTime] = new Date();
         }
     }
 }

package/dist/commonjs/index.d.ts

@@ -1,4 +1,6 @@
+import { HttpClient } from './HttpClient.js';
 import { RequestOptions, RequestURL } from './Request.js';
+export declare function getDefaultHttpClient(): HttpClient;
 export declare function request<T = any>(url: RequestURL, options?: RequestOptions): Promise<import("./Response.js").HttpClientResponse<T>>;
 export declare function curl<T = any>(url: RequestURL, options?: RequestOptions): Promise<import("./Response.js").HttpClientResponse<T>>;
 export { MockAgent, ProxyAgent, Agent, Dispatcher, setGlobalDispatcher, getGlobalDispatcher, } from 'undici';

package/dist/commonjs/index.js

@@ -17,26 +17,30 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.USER_AGENT = exports.HttpClient2 = exports.HttpClient = exports.getGlobalDispatcher = exports.setGlobalDispatcher = exports.Dispatcher = exports.Agent = exports.ProxyAgent = exports.MockAgent = exports.curl = exports.request = void 0;
+exports.USER_AGENT = exports.HttpClient2 = exports.HttpClient = exports.getGlobalDispatcher = exports.setGlobalDispatcher = exports.Dispatcher = exports.Agent = exports.ProxyAgent = exports.MockAgent = exports.curl = exports.request = exports.getDefaultHttpClient = void 0;
 const ylru_1 = __importDefault(require("ylru"));
 const HttpClient_js_1 = require("./HttpClient.js");
-let httpclient;
-const domainSocketHttpclients = new ylru_1.default(50);
+let httpClient;
+const domainSocketHttpClients = new ylru_1.default(50);
+function getDefaultHttpClient() {
+    if (!httpClient) {
+        httpClient = new HttpClient_js_1.HttpClient();
+    }
+    return httpClient;
+}
+exports.getDefaultHttpClient = getDefaultHttpClient;
 async function request(url, options) {
     if (options?.socketPath) {
-        let domainSocketHttpclient = domainSocketHttpclients.get(options.socketPath);
+        let domainSocketHttpclient = domainSocketHttpClients.get(options.socketPath);
         if (!domainSocketHttpclient) {
             domainSocketHttpclient = new HttpClient_js_1.HttpClient({
                 connect: { socketPath: options.socketPath },
             });
-            domainSocketHttpclients.set(options.socketPath, domainSocketHttpclient);
+            domainSocketHttpClients.set(options.socketPath, domainSocketHttpclient);
         }
         return await domainSocketHttpclient.request(url, options);
     }
-    if (!httpclient) {
-        httpclient = new HttpClient_js_1.HttpClient({});
-    }
-    return await httpclient.request(url, options);
+    return await getDefaultHttpClient().request(url, options);
 }
 exports.request = request;
 // export curl method is keep compatible with urllib.curl()
@@ -55,7 +59,7 @@ Object.defineProperty(exports, "Agent", { enumerable: true, get: function () { r
 Object.defineProperty(exports, "Dispatcher", { enumerable: true, get: function () { return undici_1.Dispatcher; } });
 Object.defineProperty(exports, "setGlobalDispatcher", { enumerable: true, get: function () { return undici_1.setGlobalDispatcher; } });
 Object.defineProperty(exports, "getGlobalDispatcher", { enumerable: true, get: function () { return undici_1.getGlobalDispatcher; } });
-// HttpClient2 is keep compatible with urlib@2 HttpClient2
+// HttpClient2 is keep compatible with urllib@2 HttpClient2
 var HttpClient_js_2 = require("./HttpClient.js");
 Object.defineProperty(exports, "HttpClient", { enumerable: true, get: function () { return HttpClient_js_2.HttpClient; } });
 Object.defineProperty(exports, "HttpClient2", { enumerable: true, get: function () { return HttpClient_js_2.HttpClient; } });

package/dist/esm/HttpClient.d.ts

@@ -42,6 +42,11 @@ export type ClientOptions = {
     };
 };
 export declare const HEADER_USER_AGENT: string;
+export type RequestContext = {
+    retries: number;
+    socketErrorRetries: number;
+    requestStartTime?: number;
+};
 export type RequestDiagnosticsMessage = {
     request: RequestMeta;
 };
@@ -50,11 +55,26 @@ export type ResponseDiagnosticsMessage = {
     response: RawResponseWithMeta;
     error?: Error;
 };
+export interface PoolStat {
+    /** Number of open socket connections in this pool. */
+    connected: number;
+    /** Number of open socket connections in this pool that do not have an active request. */
+    free: number;
+    /** Number of pending requests across all clients in this pool. */
+    pending: number;
+    /** Number of queued requests across all clients in this pool. */
+    queued: number;
+    /** Number of currently active requests across all clients in this pool. */
+    running: number;
+    /** Number of active, pending, or queued requests across all clients in this pool. */
+    size: number;
+}
 export declare class HttpClient extends EventEmitter {
     #private;
     constructor(clientOptions?: ClientOptions);
     getDispatcher(): Dispatcher;
     setDispatcher(dispatcher: Dispatcher): void;
+    getDispatcherPoolStats(): Record<string, PoolStat>;
     request<T = any>(url: RequestURL, options?: RequestOptions): Promise<HttpClientResponse<T>>;
     curl<T = any>(url: RequestURL, options?: RequestOptions): Promise<HttpClientResponse<T>>;
 }

package/dist/esm/HttpClient.js

@@ -11,6 +11,7 @@ import { createReadStream } from 'node:fs';
 import { format as urlFormat } from 'node:url';
 import { performance } from 'node:perf_hooks';
 import { FormData as FormDataNative, request as undiciRequest, Agent, getGlobalDispatcher, } from 'undici';
+import undiciSymbols from 'undici/lib/core/symbols.js';
 import { FormData as FormDataNode } from 'formdata-node';
 import { FormDataEncoder } from 'form-data-encoder';
 import createUserAgent from 'default-user-agent';
@@ -60,7 +61,7 @@ class BlobFromStream {
         return 'Blob';
     }
 }
-export const HEADER_USER_AGENT = createUserAgent('node-urllib', '3.21.0');
+export const HEADER_USER_AGENT = createUserAgent('node-urllib', '3.22.0');
 function getFileName(stream) {
     const filePath = stream.path;
     if (filePath) {
@@ -101,10 +102,31 @@ export class HttpClient extends EventEmitter {
     setDispatcher(dispatcher) {
         this.#dispatcher = dispatcher;
     }
+    getDispatcherPoolStats() {
+        const agent = this.getDispatcher();
+        // origin => Pool Instance
+        const clients = agent[undiciSymbols.kClients];
+        const poolStatsMap = {};
+        for (const [key, ref] of clients) {
+            const pool = ref.deref();
+            const stats = pool?.stats;
+            if (!stats)
+                continue;
+            poolStatsMap[key] = {
+                connected: stats.connected,
+                free: stats.free,
+                pending: stats.pending,
+                queued: stats.queued,
+                running: stats.running,
+                size: stats.size,
+            };
+        }
+        return poolStatsMap;
+    }
     async request(url, options) {
         return await this.#requestInternal(url, options);
     }
-    // alias to request, keep compatible with urlib@2 HttpClient.curl
+    // alias to request, keep compatible with urllib@2 HttpClient.curl
     async curl(url, options) {
         return await this.request(url, options);
     }

package/dist/esm/index.d.ts

@@ -1,4 +1,6 @@
+import { HttpClient } from './HttpClient.js';
 import { RequestOptions, RequestURL } from './Request.js';
+export declare function getDefaultHttpClient(): HttpClient;
 export declare function request<T = any>(url: RequestURL, options?: RequestOptions): Promise<import("./Response.js").HttpClientResponse<T>>;
 export declare function curl<T = any>(url: RequestURL, options?: RequestOptions): Promise<import("./Response.js").HttpClientResponse<T>>;
 export { MockAgent, ProxyAgent, Agent, Dispatcher, setGlobalDispatcher, getGlobalDispatcher, } from 'undici';

package/dist/esm/index.js

@@ -1,22 +1,25 @@
 import LRU from 'ylru';
 import { HttpClient, HEADER_USER_AGENT } from './HttpClient.js';
-let httpclient;
-const domainSocketHttpclients = new LRU(50);
+let httpClient;
+const domainSocketHttpClients = new LRU(50);
+export function getDefaultHttpClient() {
+    if (!httpClient) {
+        httpClient = new HttpClient();
+    }
+    return httpClient;
+}
 export async function request(url, options) {
     if (options?.socketPath) {
-        let domainSocketHttpclient = domainSocketHttpclients.get(options.socketPath);
+        let domainSocketHttpclient = domainSocketHttpClients.get(options.socketPath);
         if (!domainSocketHttpclient) {
             domainSocketHttpclient = new HttpClient({
                 connect: { socketPath: options.socketPath },
             });
-            domainSocketHttpclients.set(options.socketPath, domainSocketHttpclient);
+            domainSocketHttpClients.set(options.socketPath, domainSocketHttpclient);
         }
         return await domainSocketHttpclient.request(url, options);
     }
-    if (!httpclient) {
-        httpclient = new HttpClient({});
-    }
-    return await httpclient.request(url, options);
+    return await getDefaultHttpClient().request(url, options);
 }
 // export curl method is keep compatible with urllib.curl()
 // ```ts
@@ -27,7 +30,7 @@ export async function curl(url, options) {
     return await request(url, options);
 }
 export { MockAgent, ProxyAgent, Agent, Dispatcher, setGlobalDispatcher, getGlobalDispatcher, } from 'undici';
-// HttpClient2 is keep compatible with urlib@2 HttpClient2
+// HttpClient2 is keep compatible with urllib@2 HttpClient2
 export { HttpClient, HttpClient as HttpClient2, HEADER_USER_AGENT as USER_AGENT, } from './HttpClient.js';
 export * from './HttpClientError.js';
 export default {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "urllib",
-  "version": "3.21.0",
+  "version": "3.22.0",
   "publishConfig": {
     "tag": "latest"
   },
@@ -69,7 +69,7 @@
     "@types/qs": "^6.9.7",
     "@types/selfsigned": "^2.0.1",
     "@types/tar-stream": "^2.2.2",
-    "@vitest/coverage-v8": "beta",
+    "@vitest/coverage-v8": "^1.0.1",
     "busboy": "^1.6.0",
     "cross-env": "^7.0.3",
     "eslint": "^8.25.0",
@@ -82,7 +82,7 @@
     "tshy": "^1.0.0",
     "tshy-after": "^1.0.0",
     "typescript": "^5.0.4",
-    "vitest": "beta"
+    "vitest": "^1.0.1"
   },
   "engines": {
     "node": ">= 14.19.3"

package/src/HttpClient.ts

@@ -22,7 +22,9 @@ import {
   Dispatcher,
   Agent,
   getGlobalDispatcher,
+  Pool,
 } from 'undici';
+import undiciSymbols from 'undici/lib/core/symbols.js';
 import { FormData as FormDataNode } from 'formdata-node';
 import { FormDataEncoder } from 'form-data-encoder';
 import createUserAgent from 'default-user-agent';
@@ -136,7 +138,7 @@ function defaultIsRetry(response: HttpClientResponse) {
   return response.status >= 500;
 }
 
-type RequestContext = {
+export type RequestContext = {
   retries: number;
   socketErrorRetries: number;
   requestStartTime?: number;
@@ -157,6 +159,20 @@ export type ResponseDiagnosticsMessage = {
   error?: Error;
 };
 
+export interface PoolStat {
+  /** Number of open socket connections in this pool. */
+  connected: number;
+  /** Number of open socket connections in this pool that do not have an active request. */
+  free: number;
+  /** Number of pending requests across all clients in this pool. */
+  pending: number;
+  /** Number of queued requests across all clients in this pool. */
+  queued: number;
+  /** Number of currently active requests across all clients in this pool. */
+  running: number;
+  /** Number of active, pending, or queued requests across all clients in this pool. */
+  size: number;
+}
 
 export class HttpClient extends EventEmitter {
   #defaultArgs?: RequestOptions;
@@ -187,11 +203,32 @@ export class HttpClient extends EventEmitter {
     this.#dispatcher = dispatcher;
   }
 
+  getDispatcherPoolStats() {
+    const agent = this.getDispatcher();
+    // origin => Pool Instance
+    const clients: Map<string, WeakRef<Pool>> = agent[undiciSymbols.kClients];
+    const poolStatsMap: Record<string, PoolStat> = {};
+    for (const [ key, ref ] of clients) {
+      const pool = ref.deref();
+      const stats = pool?.stats;
+      if (!stats) continue;
+      poolStatsMap[key] = {
+        connected: stats.connected,
+        free: stats.free,
+        pending: stats.pending,
+        queued: stats.queued,
+        running: stats.running,
+        size: stats.size,
+      } satisfies PoolStat;
+    }
+    return poolStatsMap;
+  }
+
   async request<T = any>(url: RequestURL, options?: RequestOptions) {
     return await this.#requestInternal<T>(url, options);
   }
 
-  // alias to request, keep compatible with urlib@2 HttpClient.curl
+  // alias to request, keep compatible with urllib@2 HttpClient.curl
   async curl<T = any>(url: RequestURL, options?: RequestOptions) {
     return await this.request<T>(url, options);
   }

package/src/index.ts

@@ -2,25 +2,29 @@ import LRU from 'ylru';
 import { HttpClient, HEADER_USER_AGENT } from './HttpClient.js';
 import { RequestOptions, RequestURL } from './Request.js';
 
-let httpclient: HttpClient;
-const domainSocketHttpclients = new LRU(50);
+let httpClient: HttpClient;
+const domainSocketHttpClients = new LRU(50);
+
+export function getDefaultHttpClient(): HttpClient {
+  if (!httpClient) {
+    httpClient = new HttpClient();
+  }
+  return httpClient;
+}
 
 export async function request<T = any>(url: RequestURL, options?: RequestOptions) {
   if (options?.socketPath) {
-    let domainSocketHttpclient = domainSocketHttpclients.get<HttpClient>(options.socketPath);
+    let domainSocketHttpclient = domainSocketHttpClients.get<HttpClient>(options.socketPath);
     if (!domainSocketHttpclient) {
       domainSocketHttpclient = new HttpClient({
         connect: { socketPath: options.socketPath },
       });
-      domainSocketHttpclients.set(options.socketPath, domainSocketHttpclient);
+      domainSocketHttpClients.set(options.socketPath, domainSocketHttpclient);
     }
     return await domainSocketHttpclient.request<T>(url, options);
   }
 
-  if (!httpclient) {
-    httpclient = new HttpClient({});
-  }
-  return await httpclient.request<T>(url, options);
+  return await getDefaultHttpClient().request<T>(url, options);
 }
 
 // export curl method is keep compatible with urllib.curl()
@@ -36,12 +40,12 @@ export {
   MockAgent, ProxyAgent, Agent, Dispatcher,
   setGlobalDispatcher, getGlobalDispatcher,
 } from 'undici';
-// HttpClient2 is keep compatible with urlib@2 HttpClient2
+// HttpClient2 is keep compatible with urllib@2 HttpClient2
 export {
   HttpClient, HttpClient as HttpClient2, HEADER_USER_AGENT as USER_AGENT,
   RequestDiagnosticsMessage, ResponseDiagnosticsMessage,
 } from './HttpClient.js';
-// RequestOptions2 is keep compatible with urlib@2 RequestOptions2
+// RequestOptions2 is keep compatible with urllib@2 RequestOptions2
 export {
   RequestOptions, RequestOptions as RequestOptions2, RequestURL, HttpMethod,
   FixJSONCtlCharsHandler, FixJSONCtlChars,