urllib 3.21.0 → 3.22.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commonjs/HttpClient.d.ts +20 -0
- package/dist/commonjs/HttpClient.js +50 -25
- package/dist/commonjs/index.d.ts +2 -0
- package/dist/commonjs/index.js +14 -10
- package/dist/esm/HttpClient.d.ts +20 -0
- package/dist/esm/HttpClient.js +27 -2
- package/dist/esm/index.d.ts +2 -0
- package/dist/esm/index.js +12 -9
- package/package.json +3 -3
- package/src/HttpClient.ts +42 -2
- package/src/index.ts +14 -10
@@ -42,6 +42,11 @@ export type ClientOptions = {
|
|
42
42
|
};
|
43
43
|
};
|
44
44
|
export declare const HEADER_USER_AGENT: string;
|
45
|
+
export type RequestContext = {
|
46
|
+
retries: number;
|
47
|
+
socketErrorRetries: number;
|
48
|
+
requestStartTime?: number;
|
49
|
+
};
|
45
50
|
export type RequestDiagnosticsMessage = {
|
46
51
|
request: RequestMeta;
|
47
52
|
};
|
@@ -50,11 +55,26 @@ export type ResponseDiagnosticsMessage = {
|
|
50
55
|
response: RawResponseWithMeta;
|
51
56
|
error?: Error;
|
52
57
|
};
|
58
|
+
export interface PoolStat {
|
59
|
+
/** Number of open socket connections in this pool. */
|
60
|
+
connected: number;
|
61
|
+
/** Number of open socket connections in this pool that do not have an active request. */
|
62
|
+
free: number;
|
63
|
+
/** Number of pending requests across all clients in this pool. */
|
64
|
+
pending: number;
|
65
|
+
/** Number of queued requests across all clients in this pool. */
|
66
|
+
queued: number;
|
67
|
+
/** Number of currently active requests across all clients in this pool. */
|
68
|
+
running: number;
|
69
|
+
/** Number of active, pending, or queued requests across all clients in this pool. */
|
70
|
+
size: number;
|
71
|
+
}
|
53
72
|
export declare class HttpClient extends EventEmitter {
|
54
73
|
#private;
|
55
74
|
constructor(clientOptions?: ClientOptions);
|
56
75
|
getDispatcher(): Dispatcher;
|
57
76
|
setDispatcher(dispatcher: Dispatcher): void;
|
77
|
+
getDispatcherPoolStats(): Record<string, PoolStat>;
|
58
78
|
request<T = any>(url: RequestURL, options?: RequestOptions): Promise<HttpClientResponse<T>>;
|
59
79
|
curl<T = any>(url: RequestURL, options?: RequestOptions): Promise<HttpClientResponse<T>>;
|
60
80
|
}
|
@@ -17,6 +17,7 @@ const node_fs_1 = require("node:fs");
|
|
17
17
|
const node_url_1 = require("node:url");
|
18
18
|
const node_perf_hooks_1 = require("node:perf_hooks");
|
19
19
|
const undici_1 = require("undici");
|
20
|
+
const symbols_js_1 = __importDefault(require("undici/lib/core/symbols.js"));
|
20
21
|
const formdata_node_1 = require("formdata-node");
|
21
22
|
const form_data_encoder_1 = require("form-data-encoder");
|
22
23
|
const default_user_agent_1 = __importDefault(require("default-user-agent"));
|
@@ -27,7 +28,7 @@ const pump_1 = __importDefault(require("pump"));
|
|
27
28
|
const formstream_1 = __importDefault(require("formstream"));
|
28
29
|
const HttpAgent_js_1 = require("./HttpAgent.js");
|
29
30
|
const utils_js_1 = require("./utils.js");
|
30
|
-
const
|
31
|
+
const symbols_js_2 = __importDefault(require("./symbols.js"));
|
31
32
|
const diagnosticsChannel_js_1 = require("./diagnosticsChannel.js");
|
32
33
|
const HttpClientError_js_1 = require("./HttpClientError.js");
|
33
34
|
const PROTO_RE = /^https?:\/\//i;
|
@@ -66,7 +67,7 @@ class BlobFromStream {
|
|
66
67
|
return 'Blob';
|
67
68
|
}
|
68
69
|
}
|
69
|
-
exports.HEADER_USER_AGENT = (0, default_user_agent_1.default)('node-urllib', '3.
|
70
|
+
exports.HEADER_USER_AGENT = (0, default_user_agent_1.default)('node-urllib', '3.22.1');
|
70
71
|
function getFileName(stream) {
|
71
72
|
const filePath = stream.path;
|
72
73
|
if (filePath) {
|
@@ -107,10 +108,34 @@ class HttpClient extends node_events_1.EventEmitter {
|
|
107
108
|
setDispatcher(dispatcher) {
|
108
109
|
this.#dispatcher = dispatcher;
|
109
110
|
}
|
111
|
+
getDispatcherPoolStats() {
|
112
|
+
const agent = this.getDispatcher();
|
113
|
+
// origin => Pool Instance
|
114
|
+
const clients = agent[symbols_js_1.default.kClients];
|
115
|
+
const poolStatsMap = {};
|
116
|
+
if (!clients) {
|
117
|
+
return poolStatsMap;
|
118
|
+
}
|
119
|
+
for (const [key, ref] of clients) {
|
120
|
+
const pool = ref.deref();
|
121
|
+
const stats = pool?.stats;
|
122
|
+
if (!stats)
|
123
|
+
continue;
|
124
|
+
poolStatsMap[key] = {
|
125
|
+
connected: stats.connected,
|
126
|
+
free: stats.free,
|
127
|
+
pending: stats.pending,
|
128
|
+
queued: stats.queued,
|
129
|
+
running: stats.running,
|
130
|
+
size: stats.size,
|
131
|
+
};
|
132
|
+
}
|
133
|
+
return poolStatsMap;
|
134
|
+
}
|
110
135
|
async request(url, options) {
|
111
136
|
return await this.#requestInternal(url, options);
|
112
137
|
}
|
113
|
-
// alias to request, keep compatible with
|
138
|
+
// alias to request, keep compatible with urllib@2 HttpClient.curl
|
114
139
|
async curl(url, options) {
|
115
140
|
return await this.request(url, options);
|
116
141
|
}
|
@@ -176,11 +201,11 @@ class HttpClient extends node_events_1.EventEmitter {
|
|
176
201
|
const originalOpaque = args.opaque;
|
177
202
|
// using opaque to diagnostics channel, binding request and socket
|
178
203
|
const internalOpaque = {
|
179
|
-
[
|
180
|
-
[
|
181
|
-
[
|
182
|
-
[
|
183
|
-
[
|
204
|
+
[symbols_js_2.default.kRequestId]: requestId,
|
205
|
+
[symbols_js_2.default.kRequestStartTime]: requestStartTime,
|
206
|
+
[symbols_js_2.default.kEnableRequestTiming]: !!args.timing,
|
207
|
+
[symbols_js_2.default.kRequestTiming]: timing,
|
208
|
+
[symbols_js_2.default.kRequestOriginalOpaque]: originalOpaque,
|
184
209
|
};
|
185
210
|
const reqMeta = {
|
186
211
|
requestId,
|
@@ -628,14 +653,14 @@ class HttpClient extends node_events_1.EventEmitter {
|
|
628
653
|
}
|
629
654
|
}
|
630
655
|
#updateSocketInfo(socketInfo, internalOpaque, err) {
|
631
|
-
const socket = internalOpaque[
|
656
|
+
const socket = internalOpaque[symbols_js_2.default.kRequestSocket] ?? err?.[symbols_js_2.default.kErrorSocket];
|
632
657
|
if (socket) {
|
633
|
-
socketInfo.id = socket[
|
634
|
-
socketInfo.handledRequests = socket[
|
635
|
-
socketInfo.handledResponses = socket[
|
636
|
-
if (socket[
|
637
|
-
socketInfo.localAddress = socket[
|
638
|
-
socketInfo.localPort = socket[
|
658
|
+
socketInfo.id = socket[symbols_js_2.default.kSocketId];
|
659
|
+
socketInfo.handledRequests = socket[symbols_js_2.default.kHandledRequests];
|
660
|
+
socketInfo.handledResponses = socket[symbols_js_2.default.kHandledResponses];
|
661
|
+
if (socket[symbols_js_2.default.kSocketLocalAddress]) {
|
662
|
+
socketInfo.localAddress = socket[symbols_js_2.default.kSocketLocalAddress];
|
663
|
+
socketInfo.localPort = socket[symbols_js_2.default.kSocketLocalPort];
|
639
664
|
}
|
640
665
|
if (socket.remoteAddress) {
|
641
666
|
socketInfo.remoteAddress = socket.remoteAddress;
|
@@ -644,22 +669,22 @@ class HttpClient extends node_events_1.EventEmitter {
|
|
644
669
|
}
|
645
670
|
socketInfo.bytesRead = socket.bytesRead;
|
646
671
|
socketInfo.bytesWritten = socket.bytesWritten;
|
647
|
-
if (socket[
|
648
|
-
socketInfo.connectErrorTime = socket[
|
672
|
+
if (socket[symbols_js_2.default.kSocketConnectErrorTime]) {
|
673
|
+
socketInfo.connectErrorTime = socket[symbols_js_2.default.kSocketConnectErrorTime];
|
649
674
|
if (Array.isArray(socket.autoSelectFamilyAttemptedAddresses)) {
|
650
675
|
socketInfo.attemptedRemoteAddresses = socket.autoSelectFamilyAttemptedAddresses;
|
651
676
|
}
|
652
|
-
socketInfo.connectProtocol = socket[
|
653
|
-
socketInfo.connectHost = socket[
|
654
|
-
socketInfo.connectPort = socket[
|
677
|
+
socketInfo.connectProtocol = socket[symbols_js_2.default.kSocketConnectProtocol];
|
678
|
+
socketInfo.connectHost = socket[symbols_js_2.default.kSocketConnectHost];
|
679
|
+
socketInfo.connectPort = socket[symbols_js_2.default.kSocketConnectPort];
|
655
680
|
}
|
656
|
-
if (socket[
|
657
|
-
socketInfo.connectedTime = socket[
|
681
|
+
if (socket[symbols_js_2.default.kSocketConnectedTime]) {
|
682
|
+
socketInfo.connectedTime = socket[symbols_js_2.default.kSocketConnectedTime];
|
658
683
|
}
|
659
|
-
if (socket[
|
660
|
-
socketInfo.lastRequestEndTime = socket[
|
684
|
+
if (socket[symbols_js_2.default.kSocketRequestEndTime]) {
|
685
|
+
socketInfo.lastRequestEndTime = socket[symbols_js_2.default.kSocketRequestEndTime];
|
661
686
|
}
|
662
|
-
socket[
|
687
|
+
socket[symbols_js_2.default.kSocketRequestEndTime] = new Date();
|
663
688
|
}
|
664
689
|
}
|
665
690
|
}
|
package/dist/commonjs/index.d.ts
CHANGED
@@ -1,4 +1,6 @@
|
|
1
|
+
import { HttpClient } from './HttpClient.js';
|
1
2
|
import { RequestOptions, RequestURL } from './Request.js';
|
3
|
+
export declare function getDefaultHttpClient(): HttpClient;
|
2
4
|
export declare function request<T = any>(url: RequestURL, options?: RequestOptions): Promise<import("./Response.js").HttpClientResponse<T>>;
|
3
5
|
export declare function curl<T = any>(url: RequestURL, options?: RequestOptions): Promise<import("./Response.js").HttpClientResponse<T>>;
|
4
6
|
export { MockAgent, ProxyAgent, Agent, Dispatcher, setGlobalDispatcher, getGlobalDispatcher, } from 'undici';
|
package/dist/commonjs/index.js
CHANGED
@@ -17,26 +17,30 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
17
17
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
18
18
|
};
|
19
19
|
Object.defineProperty(exports, "__esModule", { value: true });
|
20
|
-
exports.USER_AGENT = exports.HttpClient2 = exports.HttpClient = exports.getGlobalDispatcher = exports.setGlobalDispatcher = exports.Dispatcher = exports.Agent = exports.ProxyAgent = exports.MockAgent = exports.curl = exports.request = void 0;
|
20
|
+
exports.USER_AGENT = exports.HttpClient2 = exports.HttpClient = exports.getGlobalDispatcher = exports.setGlobalDispatcher = exports.Dispatcher = exports.Agent = exports.ProxyAgent = exports.MockAgent = exports.curl = exports.request = exports.getDefaultHttpClient = void 0;
|
21
21
|
const ylru_1 = __importDefault(require("ylru"));
|
22
22
|
const HttpClient_js_1 = require("./HttpClient.js");
|
23
|
-
let
|
24
|
-
const
|
23
|
+
let httpClient;
|
24
|
+
const domainSocketHttpClients = new ylru_1.default(50);
|
25
|
+
function getDefaultHttpClient() {
|
26
|
+
if (!httpClient) {
|
27
|
+
httpClient = new HttpClient_js_1.HttpClient();
|
28
|
+
}
|
29
|
+
return httpClient;
|
30
|
+
}
|
31
|
+
exports.getDefaultHttpClient = getDefaultHttpClient;
|
25
32
|
async function request(url, options) {
|
26
33
|
if (options?.socketPath) {
|
27
|
-
let domainSocketHttpclient =
|
34
|
+
let domainSocketHttpclient = domainSocketHttpClients.get(options.socketPath);
|
28
35
|
if (!domainSocketHttpclient) {
|
29
36
|
domainSocketHttpclient = new HttpClient_js_1.HttpClient({
|
30
37
|
connect: { socketPath: options.socketPath },
|
31
38
|
});
|
32
|
-
|
39
|
+
domainSocketHttpClients.set(options.socketPath, domainSocketHttpclient);
|
33
40
|
}
|
34
41
|
return await domainSocketHttpclient.request(url, options);
|
35
42
|
}
|
36
|
-
|
37
|
-
httpclient = new HttpClient_js_1.HttpClient({});
|
38
|
-
}
|
39
|
-
return await httpclient.request(url, options);
|
43
|
+
return await getDefaultHttpClient().request(url, options);
|
40
44
|
}
|
41
45
|
exports.request = request;
|
42
46
|
// export curl method is keep compatible with urllib.curl()
|
@@ -55,7 +59,7 @@ Object.defineProperty(exports, "Agent", { enumerable: true, get: function () { r
|
|
55
59
|
Object.defineProperty(exports, "Dispatcher", { enumerable: true, get: function () { return undici_1.Dispatcher; } });
|
56
60
|
Object.defineProperty(exports, "setGlobalDispatcher", { enumerable: true, get: function () { return undici_1.setGlobalDispatcher; } });
|
57
61
|
Object.defineProperty(exports, "getGlobalDispatcher", { enumerable: true, get: function () { return undici_1.getGlobalDispatcher; } });
|
58
|
-
// HttpClient2 is keep compatible with
|
62
|
+
// HttpClient2 is keep compatible with urllib@2 HttpClient2
|
59
63
|
var HttpClient_js_2 = require("./HttpClient.js");
|
60
64
|
Object.defineProperty(exports, "HttpClient", { enumerable: true, get: function () { return HttpClient_js_2.HttpClient; } });
|
61
65
|
Object.defineProperty(exports, "HttpClient2", { enumerable: true, get: function () { return HttpClient_js_2.HttpClient; } });
|
package/dist/esm/HttpClient.d.ts
CHANGED
@@ -42,6 +42,11 @@ export type ClientOptions = {
|
|
42
42
|
};
|
43
43
|
};
|
44
44
|
export declare const HEADER_USER_AGENT: string;
|
45
|
+
export type RequestContext = {
|
46
|
+
retries: number;
|
47
|
+
socketErrorRetries: number;
|
48
|
+
requestStartTime?: number;
|
49
|
+
};
|
45
50
|
export type RequestDiagnosticsMessage = {
|
46
51
|
request: RequestMeta;
|
47
52
|
};
|
@@ -50,11 +55,26 @@ export type ResponseDiagnosticsMessage = {
|
|
50
55
|
response: RawResponseWithMeta;
|
51
56
|
error?: Error;
|
52
57
|
};
|
58
|
+
export interface PoolStat {
|
59
|
+
/** Number of open socket connections in this pool. */
|
60
|
+
connected: number;
|
61
|
+
/** Number of open socket connections in this pool that do not have an active request. */
|
62
|
+
free: number;
|
63
|
+
/** Number of pending requests across all clients in this pool. */
|
64
|
+
pending: number;
|
65
|
+
/** Number of queued requests across all clients in this pool. */
|
66
|
+
queued: number;
|
67
|
+
/** Number of currently active requests across all clients in this pool. */
|
68
|
+
running: number;
|
69
|
+
/** Number of active, pending, or queued requests across all clients in this pool. */
|
70
|
+
size: number;
|
71
|
+
}
|
53
72
|
export declare class HttpClient extends EventEmitter {
|
54
73
|
#private;
|
55
74
|
constructor(clientOptions?: ClientOptions);
|
56
75
|
getDispatcher(): Dispatcher;
|
57
76
|
setDispatcher(dispatcher: Dispatcher): void;
|
77
|
+
getDispatcherPoolStats(): Record<string, PoolStat>;
|
58
78
|
request<T = any>(url: RequestURL, options?: RequestOptions): Promise<HttpClientResponse<T>>;
|
59
79
|
curl<T = any>(url: RequestURL, options?: RequestOptions): Promise<HttpClientResponse<T>>;
|
60
80
|
}
|
package/dist/esm/HttpClient.js
CHANGED
@@ -11,6 +11,7 @@ import { createReadStream } from 'node:fs';
|
|
11
11
|
import { format as urlFormat } from 'node:url';
|
12
12
|
import { performance } from 'node:perf_hooks';
|
13
13
|
import { FormData as FormDataNative, request as undiciRequest, Agent, getGlobalDispatcher, } from 'undici';
|
14
|
+
import undiciSymbols from 'undici/lib/core/symbols.js';
|
14
15
|
import { FormData as FormDataNode } from 'formdata-node';
|
15
16
|
import { FormDataEncoder } from 'form-data-encoder';
|
16
17
|
import createUserAgent from 'default-user-agent';
|
@@ -60,7 +61,7 @@ class BlobFromStream {
|
|
60
61
|
return 'Blob';
|
61
62
|
}
|
62
63
|
}
|
63
|
-
export const HEADER_USER_AGENT = createUserAgent('node-urllib', '3.
|
64
|
+
export const HEADER_USER_AGENT = createUserAgent('node-urllib', '3.22.1');
|
64
65
|
function getFileName(stream) {
|
65
66
|
const filePath = stream.path;
|
66
67
|
if (filePath) {
|
@@ -101,10 +102,34 @@ export class HttpClient extends EventEmitter {
|
|
101
102
|
setDispatcher(dispatcher) {
|
102
103
|
this.#dispatcher = dispatcher;
|
103
104
|
}
|
105
|
+
getDispatcherPoolStats() {
|
106
|
+
const agent = this.getDispatcher();
|
107
|
+
// origin => Pool Instance
|
108
|
+
const clients = agent[undiciSymbols.kClients];
|
109
|
+
const poolStatsMap = {};
|
110
|
+
if (!clients) {
|
111
|
+
return poolStatsMap;
|
112
|
+
}
|
113
|
+
for (const [key, ref] of clients) {
|
114
|
+
const pool = ref.deref();
|
115
|
+
const stats = pool?.stats;
|
116
|
+
if (!stats)
|
117
|
+
continue;
|
118
|
+
poolStatsMap[key] = {
|
119
|
+
connected: stats.connected,
|
120
|
+
free: stats.free,
|
121
|
+
pending: stats.pending,
|
122
|
+
queued: stats.queued,
|
123
|
+
running: stats.running,
|
124
|
+
size: stats.size,
|
125
|
+
};
|
126
|
+
}
|
127
|
+
return poolStatsMap;
|
128
|
+
}
|
104
129
|
async request(url, options) {
|
105
130
|
return await this.#requestInternal(url, options);
|
106
131
|
}
|
107
|
-
// alias to request, keep compatible with
|
132
|
+
// alias to request, keep compatible with urllib@2 HttpClient.curl
|
108
133
|
async curl(url, options) {
|
109
134
|
return await this.request(url, options);
|
110
135
|
}
|
package/dist/esm/index.d.ts
CHANGED
@@ -1,4 +1,6 @@
|
|
1
|
+
import { HttpClient } from './HttpClient.js';
|
1
2
|
import { RequestOptions, RequestURL } from './Request.js';
|
3
|
+
export declare function getDefaultHttpClient(): HttpClient;
|
2
4
|
export declare function request<T = any>(url: RequestURL, options?: RequestOptions): Promise<import("./Response.js").HttpClientResponse<T>>;
|
3
5
|
export declare function curl<T = any>(url: RequestURL, options?: RequestOptions): Promise<import("./Response.js").HttpClientResponse<T>>;
|
4
6
|
export { MockAgent, ProxyAgent, Agent, Dispatcher, setGlobalDispatcher, getGlobalDispatcher, } from 'undici';
|
package/dist/esm/index.js
CHANGED
@@ -1,22 +1,25 @@
|
|
1
1
|
import LRU from 'ylru';
|
2
2
|
import { HttpClient, HEADER_USER_AGENT } from './HttpClient.js';
|
3
|
-
let
|
4
|
-
const
|
3
|
+
let httpClient;
|
4
|
+
const domainSocketHttpClients = new LRU(50);
|
5
|
+
export function getDefaultHttpClient() {
|
6
|
+
if (!httpClient) {
|
7
|
+
httpClient = new HttpClient();
|
8
|
+
}
|
9
|
+
return httpClient;
|
10
|
+
}
|
5
11
|
export async function request(url, options) {
|
6
12
|
if (options?.socketPath) {
|
7
|
-
let domainSocketHttpclient =
|
13
|
+
let domainSocketHttpclient = domainSocketHttpClients.get(options.socketPath);
|
8
14
|
if (!domainSocketHttpclient) {
|
9
15
|
domainSocketHttpclient = new HttpClient({
|
10
16
|
connect: { socketPath: options.socketPath },
|
11
17
|
});
|
12
|
-
|
18
|
+
domainSocketHttpClients.set(options.socketPath, domainSocketHttpclient);
|
13
19
|
}
|
14
20
|
return await domainSocketHttpclient.request(url, options);
|
15
21
|
}
|
16
|
-
|
17
|
-
httpclient = new HttpClient({});
|
18
|
-
}
|
19
|
-
return await httpclient.request(url, options);
|
22
|
+
return await getDefaultHttpClient().request(url, options);
|
20
23
|
}
|
21
24
|
// export curl method is keep compatible with urllib.curl()
|
22
25
|
// ```ts
|
@@ -27,7 +30,7 @@ export async function curl(url, options) {
|
|
27
30
|
return await request(url, options);
|
28
31
|
}
|
29
32
|
export { MockAgent, ProxyAgent, Agent, Dispatcher, setGlobalDispatcher, getGlobalDispatcher, } from 'undici';
|
30
|
-
// HttpClient2 is keep compatible with
|
33
|
+
// HttpClient2 is keep compatible with urllib@2 HttpClient2
|
31
34
|
export { HttpClient, HttpClient as HttpClient2, HEADER_USER_AGENT as USER_AGENT, } from './HttpClient.js';
|
32
35
|
export * from './HttpClientError.js';
|
33
36
|
export default {
|
package/package.json
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
{
|
2
2
|
"name": "urllib",
|
3
|
-
"version": "3.
|
3
|
+
"version": "3.22.1",
|
4
4
|
"publishConfig": {
|
5
5
|
"tag": "latest"
|
6
6
|
},
|
@@ -69,7 +69,7 @@
|
|
69
69
|
"@types/qs": "^6.9.7",
|
70
70
|
"@types/selfsigned": "^2.0.1",
|
71
71
|
"@types/tar-stream": "^2.2.2",
|
72
|
-
"@vitest/coverage-v8": "
|
72
|
+
"@vitest/coverage-v8": "^1.0.1",
|
73
73
|
"busboy": "^1.6.0",
|
74
74
|
"cross-env": "^7.0.3",
|
75
75
|
"eslint": "^8.25.0",
|
@@ -82,7 +82,7 @@
|
|
82
82
|
"tshy": "^1.0.0",
|
83
83
|
"tshy-after": "^1.0.0",
|
84
84
|
"typescript": "^5.0.4",
|
85
|
-
"vitest": "
|
85
|
+
"vitest": "^1.0.1"
|
86
86
|
},
|
87
87
|
"engines": {
|
88
88
|
"node": ">= 14.19.3"
|
package/src/HttpClient.ts
CHANGED
@@ -22,7 +22,9 @@ import {
|
|
22
22
|
Dispatcher,
|
23
23
|
Agent,
|
24
24
|
getGlobalDispatcher,
|
25
|
+
Pool,
|
25
26
|
} from 'undici';
|
27
|
+
import undiciSymbols from 'undici/lib/core/symbols.js';
|
26
28
|
import { FormData as FormDataNode } from 'formdata-node';
|
27
29
|
import { FormDataEncoder } from 'form-data-encoder';
|
28
30
|
import createUserAgent from 'default-user-agent';
|
@@ -136,7 +138,7 @@ function defaultIsRetry(response: HttpClientResponse) {
|
|
136
138
|
return response.status >= 500;
|
137
139
|
}
|
138
140
|
|
139
|
-
type RequestContext = {
|
141
|
+
export type RequestContext = {
|
140
142
|
retries: number;
|
141
143
|
socketErrorRetries: number;
|
142
144
|
requestStartTime?: number;
|
@@ -157,6 +159,20 @@ export type ResponseDiagnosticsMessage = {
|
|
157
159
|
error?: Error;
|
158
160
|
};
|
159
161
|
|
162
|
+
export interface PoolStat {
|
163
|
+
/** Number of open socket connections in this pool. */
|
164
|
+
connected: number;
|
165
|
+
/** Number of open socket connections in this pool that do not have an active request. */
|
166
|
+
free: number;
|
167
|
+
/** Number of pending requests across all clients in this pool. */
|
168
|
+
pending: number;
|
169
|
+
/** Number of queued requests across all clients in this pool. */
|
170
|
+
queued: number;
|
171
|
+
/** Number of currently active requests across all clients in this pool. */
|
172
|
+
running: number;
|
173
|
+
/** Number of active, pending, or queued requests across all clients in this pool. */
|
174
|
+
size: number;
|
175
|
+
}
|
160
176
|
|
161
177
|
export class HttpClient extends EventEmitter {
|
162
178
|
#defaultArgs?: RequestOptions;
|
@@ -187,11 +203,35 @@ export class HttpClient extends EventEmitter {
|
|
187
203
|
this.#dispatcher = dispatcher;
|
188
204
|
}
|
189
205
|
|
206
|
+
getDispatcherPoolStats() {
|
207
|
+
const agent = this.getDispatcher();
|
208
|
+
// origin => Pool Instance
|
209
|
+
const clients: Map<string, WeakRef<Pool>> | undefined = agent[undiciSymbols.kClients];
|
210
|
+
const poolStatsMap: Record<string, PoolStat> = {};
|
211
|
+
if (!clients) {
|
212
|
+
return poolStatsMap;
|
213
|
+
}
|
214
|
+
for (const [ key, ref ] of clients) {
|
215
|
+
const pool = ref.deref();
|
216
|
+
const stats = pool?.stats;
|
217
|
+
if (!stats) continue;
|
218
|
+
poolStatsMap[key] = {
|
219
|
+
connected: stats.connected,
|
220
|
+
free: stats.free,
|
221
|
+
pending: stats.pending,
|
222
|
+
queued: stats.queued,
|
223
|
+
running: stats.running,
|
224
|
+
size: stats.size,
|
225
|
+
} satisfies PoolStat;
|
226
|
+
}
|
227
|
+
return poolStatsMap;
|
228
|
+
}
|
229
|
+
|
190
230
|
async request<T = any>(url: RequestURL, options?: RequestOptions) {
|
191
231
|
return await this.#requestInternal<T>(url, options);
|
192
232
|
}
|
193
233
|
|
194
|
-
// alias to request, keep compatible with
|
234
|
+
// alias to request, keep compatible with urllib@2 HttpClient.curl
|
195
235
|
async curl<T = any>(url: RequestURL, options?: RequestOptions) {
|
196
236
|
return await this.request<T>(url, options);
|
197
237
|
}
|
package/src/index.ts
CHANGED
@@ -2,25 +2,29 @@ import LRU from 'ylru';
|
|
2
2
|
import { HttpClient, HEADER_USER_AGENT } from './HttpClient.js';
|
3
3
|
import { RequestOptions, RequestURL } from './Request.js';
|
4
4
|
|
5
|
-
let
|
6
|
-
const
|
5
|
+
let httpClient: HttpClient;
|
6
|
+
const domainSocketHttpClients = new LRU(50);
|
7
|
+
|
8
|
+
export function getDefaultHttpClient(): HttpClient {
|
9
|
+
if (!httpClient) {
|
10
|
+
httpClient = new HttpClient();
|
11
|
+
}
|
12
|
+
return httpClient;
|
13
|
+
}
|
7
14
|
|
8
15
|
export async function request<T = any>(url: RequestURL, options?: RequestOptions) {
|
9
16
|
if (options?.socketPath) {
|
10
|
-
let domainSocketHttpclient =
|
17
|
+
let domainSocketHttpclient = domainSocketHttpClients.get<HttpClient>(options.socketPath);
|
11
18
|
if (!domainSocketHttpclient) {
|
12
19
|
domainSocketHttpclient = new HttpClient({
|
13
20
|
connect: { socketPath: options.socketPath },
|
14
21
|
});
|
15
|
-
|
22
|
+
domainSocketHttpClients.set(options.socketPath, domainSocketHttpclient);
|
16
23
|
}
|
17
24
|
return await domainSocketHttpclient.request<T>(url, options);
|
18
25
|
}
|
19
26
|
|
20
|
-
|
21
|
-
httpclient = new HttpClient({});
|
22
|
-
}
|
23
|
-
return await httpclient.request<T>(url, options);
|
27
|
+
return await getDefaultHttpClient().request<T>(url, options);
|
24
28
|
}
|
25
29
|
|
26
30
|
// export curl method is keep compatible with urllib.curl()
|
@@ -36,12 +40,12 @@ export {
|
|
36
40
|
MockAgent, ProxyAgent, Agent, Dispatcher,
|
37
41
|
setGlobalDispatcher, getGlobalDispatcher,
|
38
42
|
} from 'undici';
|
39
|
-
// HttpClient2 is keep compatible with
|
43
|
+
// HttpClient2 is keep compatible with urllib@2 HttpClient2
|
40
44
|
export {
|
41
45
|
HttpClient, HttpClient as HttpClient2, HEADER_USER_AGENT as USER_AGENT,
|
42
46
|
RequestDiagnosticsMessage, ResponseDiagnosticsMessage,
|
43
47
|
} from './HttpClient.js';
|
44
|
-
// RequestOptions2 is keep compatible with
|
48
|
+
// RequestOptions2 is keep compatible with urllib@2 RequestOptions2
|
45
49
|
export {
|
46
50
|
RequestOptions, RequestOptions as RequestOptions2, RequestURL, HttpMethod,
|
47
51
|
FixJSONCtlCharsHandler, FixJSONCtlChars,
|