urllib 3.20.0 → 3.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commonjs/HttpAgent.js +1 -1
- package/dist/commonjs/HttpClient.d.ts +20 -0
- package/dist/commonjs/HttpClient.js +70 -35
- package/dist/commonjs/HttpClientError.d.ts +19 -0
- package/dist/commonjs/HttpClientError.js +29 -0
- package/dist/commonjs/Response.d.ts +5 -0
- package/dist/commonjs/diagnosticsChannel.js +45 -2
- package/dist/commonjs/index.d.ts +3 -0
- package/dist/commonjs/index.js +29 -10
- package/dist/commonjs/symbols.d.ts +5 -0
- package/dist/commonjs/symbols.js +5 -0
- package/dist/esm/HttpAgent.js +1 -1
- package/dist/esm/HttpClient.d.ts +20 -0
- package/dist/esm/HttpClient.js +60 -25
- package/dist/esm/HttpClientError.d.ts +19 -0
- package/dist/esm/HttpClientError.js +23 -0
- package/dist/esm/Response.d.ts +5 -0
- package/dist/esm/diagnosticsChannel.js +45 -2
- package/dist/esm/index.d.ts +3 -0
- package/dist/esm/index.js +13 -9
- package/dist/esm/symbols.d.ts +5 -0
- package/dist/esm/symbols.js +5 -0
- package/package.json +5 -4
- package/src/HttpAgent.ts +1 -1
- package/src/HttpClient.ts +74 -26
- package/src/HttpClientError.ts +34 -0
- package/src/Response.ts +5 -0
- package/src/diagnosticsChannel.ts +48 -2
- package/src/index.ts +15 -10
- package/src/symbols.ts +5 -0
package/dist/esm/HttpClient.js
CHANGED
@@ -11,6 +11,7 @@ import { createReadStream } from 'node:fs';
|
|
11
11
|
import { format as urlFormat } from 'node:url';
|
12
12
|
import { performance } from 'node:perf_hooks';
|
13
13
|
import { FormData as FormDataNative, request as undiciRequest, Agent, getGlobalDispatcher, } from 'undici';
|
14
|
+
import undiciSymbols from 'undici/lib/core/symbols.js';
|
14
15
|
import { FormData as FormDataNode } from 'formdata-node';
|
15
16
|
import { FormDataEncoder } from 'form-data-encoder';
|
16
17
|
import createUserAgent from 'default-user-agent';
|
@@ -23,6 +24,7 @@ import { HttpAgent } from './HttpAgent.js';
|
|
23
24
|
import { parseJSON, sleep, digestAuthHeader, globalId, performanceTime, isReadable } from './utils.js';
|
24
25
|
import symbols from './symbols.js';
|
25
26
|
import { initDiagnosticsChannel } from './diagnosticsChannel.js';
|
27
|
+
import { HttpClientConnectTimeoutError, HttpClientRequestTimeoutError } from './HttpClientError.js';
|
26
28
|
const PROTO_RE = /^https?:\/\//i;
|
27
29
|
const FormData = FormDataNative ?? FormDataNode;
|
28
30
|
// impl promise pipeline on Node.js 14
|
@@ -59,15 +61,7 @@ class BlobFromStream {
|
|
59
61
|
return 'Blob';
|
60
62
|
}
|
61
63
|
}
|
62
|
-
|
63
|
-
constructor(timeout, options) {
|
64
|
-
const message = `Request timeout for ${timeout} ms`;
|
65
|
-
super(message, options);
|
66
|
-
this.name = this.constructor.name;
|
67
|
-
Error.captureStackTrace(this, this.constructor);
|
68
|
-
}
|
69
|
-
}
|
70
|
-
export const HEADER_USER_AGENT = createUserAgent('node-urllib', '3.20.0');
|
64
|
+
export const HEADER_USER_AGENT = createUserAgent('node-urllib', '3.22.0');
|
71
65
|
function getFileName(stream) {
|
72
66
|
const filePath = stream.path;
|
73
67
|
if (filePath) {
|
@@ -108,10 +102,31 @@ export class HttpClient extends EventEmitter {
|
|
108
102
|
setDispatcher(dispatcher) {
|
109
103
|
this.#dispatcher = dispatcher;
|
110
104
|
}
|
105
|
+
getDispatcherPoolStats() {
|
106
|
+
const agent = this.getDispatcher();
|
107
|
+
// origin => Pool Instance
|
108
|
+
const clients = agent[undiciSymbols.kClients];
|
109
|
+
const poolStatsMap = {};
|
110
|
+
for (const [key, ref] of clients) {
|
111
|
+
const pool = ref.deref();
|
112
|
+
const stats = pool?.stats;
|
113
|
+
if (!stats)
|
114
|
+
continue;
|
115
|
+
poolStatsMap[key] = {
|
116
|
+
connected: stats.connected,
|
117
|
+
free: stats.free,
|
118
|
+
pending: stats.pending,
|
119
|
+
queued: stats.queued,
|
120
|
+
running: stats.running,
|
121
|
+
size: stats.size,
|
122
|
+
};
|
123
|
+
}
|
124
|
+
return poolStatsMap;
|
125
|
+
}
|
111
126
|
async request(url, options) {
|
112
127
|
return await this.#requestInternal(url, options);
|
113
128
|
}
|
114
|
-
// alias to request, keep compatible with
|
129
|
+
// alias to request, keep compatible with urllib@2 HttpClient.curl
|
115
130
|
async curl(url, options) {
|
116
131
|
return await this.request(url, options);
|
117
132
|
}
|
@@ -574,14 +589,17 @@ export class HttpClient extends EventEmitter {
|
|
574
589
|
}
|
575
590
|
return clientResponse;
|
576
591
|
}
|
577
|
-
catch (
|
578
|
-
debug('Request#%d throw error: %s', requestId,
|
579
|
-
let err =
|
592
|
+
catch (rawError) {
|
593
|
+
debug('Request#%d throw error: %s', requestId, rawError);
|
594
|
+
let err = rawError;
|
580
595
|
if (err.name === 'HeadersTimeoutError') {
|
581
|
-
err = new HttpClientRequestTimeoutError(headersTimeout, { cause:
|
596
|
+
err = new HttpClientRequestTimeoutError(headersTimeout, { cause: err });
|
582
597
|
}
|
583
598
|
else if (err.name === 'BodyTimeoutError') {
|
584
|
-
err = new HttpClientRequestTimeoutError(bodyTimeout, { cause:
|
599
|
+
err = new HttpClientRequestTimeoutError(bodyTimeout, { cause: err });
|
600
|
+
}
|
601
|
+
else if (err.code === 'UND_ERR_CONNECT_TIMEOUT') {
|
602
|
+
err = new HttpClientConnectTimeoutError(err.message, err.code, { cause: err });
|
585
603
|
}
|
586
604
|
else if (err.code === 'UND_ERR_SOCKET' || err.code === 'ECONNRESET') {
|
587
605
|
// auto retry on socket error, https://github.com/node-modules/urllib/issues/454
|
@@ -604,7 +622,7 @@ export class HttpClient extends EventEmitter {
|
|
604
622
|
res.requestUrls.push(requestUrl.href);
|
605
623
|
}
|
606
624
|
res.rt = performanceTime(requestStartTime);
|
607
|
-
this.#updateSocketInfo(socketInfo, internalOpaque);
|
625
|
+
this.#updateSocketInfo(socketInfo, internalOpaque, rawError);
|
608
626
|
channels.response.publish({
|
609
627
|
request: reqMeta,
|
610
628
|
response: res,
|
@@ -625,21 +643,38 @@ export class HttpClient extends EventEmitter {
|
|
625
643
|
throw err;
|
626
644
|
}
|
627
645
|
}
|
628
|
-
#updateSocketInfo(socketInfo, internalOpaque) {
|
629
|
-
const socket = internalOpaque[symbols.kRequestSocket];
|
646
|
+
#updateSocketInfo(socketInfo, internalOpaque, err) {
|
647
|
+
const socket = internalOpaque[symbols.kRequestSocket] ?? err?.[symbols.kErrorSocket];
|
630
648
|
if (socket) {
|
631
649
|
socketInfo.id = socket[symbols.kSocketId];
|
632
650
|
socketInfo.handledRequests = socket[symbols.kHandledRequests];
|
633
651
|
socketInfo.handledResponses = socket[symbols.kHandledResponses];
|
634
|
-
|
635
|
-
|
636
|
-
|
637
|
-
|
638
|
-
|
652
|
+
if (socket[symbols.kSocketLocalAddress]) {
|
653
|
+
socketInfo.localAddress = socket[symbols.kSocketLocalAddress];
|
654
|
+
socketInfo.localPort = socket[symbols.kSocketLocalPort];
|
655
|
+
}
|
656
|
+
if (socket.remoteAddress) {
|
657
|
+
socketInfo.remoteAddress = socket.remoteAddress;
|
658
|
+
socketInfo.remotePort = socket.remotePort;
|
659
|
+
socketInfo.remoteFamily = socket.remoteFamily;
|
660
|
+
}
|
639
661
|
socketInfo.bytesRead = socket.bytesRead;
|
640
662
|
socketInfo.bytesWritten = socket.bytesWritten;
|
641
|
-
|
642
|
-
|
663
|
+
if (socket[symbols.kSocketConnectErrorTime]) {
|
664
|
+
socketInfo.connectErrorTime = socket[symbols.kSocketConnectErrorTime];
|
665
|
+
if (Array.isArray(socket.autoSelectFamilyAttemptedAddresses)) {
|
666
|
+
socketInfo.attemptedRemoteAddresses = socket.autoSelectFamilyAttemptedAddresses;
|
667
|
+
}
|
668
|
+
socketInfo.connectProtocol = socket[symbols.kSocketConnectProtocol];
|
669
|
+
socketInfo.connectHost = socket[symbols.kSocketConnectHost];
|
670
|
+
socketInfo.connectPort = socket[symbols.kSocketConnectPort];
|
671
|
+
}
|
672
|
+
if (socket[symbols.kSocketConnectedTime]) {
|
673
|
+
socketInfo.connectedTime = socket[symbols.kSocketConnectedTime];
|
674
|
+
}
|
675
|
+
if (socket[symbols.kSocketRequestEndTime]) {
|
676
|
+
socketInfo.lastRequestEndTime = socket[symbols.kSocketRequestEndTime];
|
677
|
+
}
|
643
678
|
socket[symbols.kSocketRequestEndTime] = new Date();
|
644
679
|
}
|
645
680
|
}
|
@@ -0,0 +1,19 @@
|
|
1
|
+
import type { RawResponseWithMeta, SocketInfo } from './Response.js';
|
2
|
+
import type { IncomingHttpHeaders } from './IncomingHttpHeaders.js';
|
3
|
+
interface ErrorOptions {
|
4
|
+
cause?: Error;
|
5
|
+
}
|
6
|
+
export declare class HttpClientRequestError extends Error {
|
7
|
+
status?: number;
|
8
|
+
headers?: IncomingHttpHeaders;
|
9
|
+
socket?: SocketInfo;
|
10
|
+
res?: RawResponseWithMeta;
|
11
|
+
}
|
12
|
+
export declare class HttpClientRequestTimeoutError extends HttpClientRequestError {
|
13
|
+
constructor(timeout: number, options: ErrorOptions);
|
14
|
+
}
|
15
|
+
export declare class HttpClientConnectTimeoutError extends HttpClientRequestError {
|
16
|
+
code: string;
|
17
|
+
constructor(message: string, code: string, options: ErrorOptions);
|
18
|
+
}
|
19
|
+
export {};
|
@@ -0,0 +1,23 @@
|
|
1
|
+
export class HttpClientRequestError extends Error {
|
2
|
+
status;
|
3
|
+
headers;
|
4
|
+
socket;
|
5
|
+
res;
|
6
|
+
}
|
7
|
+
export class HttpClientRequestTimeoutError extends HttpClientRequestError {
|
8
|
+
constructor(timeout, options) {
|
9
|
+
const message = `Request timeout for ${timeout} ms`;
|
10
|
+
super(message, options);
|
11
|
+
this.name = this.constructor.name;
|
12
|
+
Error.captureStackTrace(this, this.constructor);
|
13
|
+
}
|
14
|
+
}
|
15
|
+
export class HttpClientConnectTimeoutError extends HttpClientRequestError {
|
16
|
+
code;
|
17
|
+
constructor(message, code, options) {
|
18
|
+
super(message, options);
|
19
|
+
this.name = this.constructor.name;
|
20
|
+
this.code = code;
|
21
|
+
Error.captureStackTrace(this, this.constructor);
|
22
|
+
}
|
23
|
+
}
|
package/dist/esm/Response.d.ts
CHANGED
@@ -13,7 +13,12 @@ export type SocketInfo = {
|
|
13
13
|
handledRequests: number;
|
14
14
|
handledResponses: number;
|
15
15
|
connectedTime?: Date;
|
16
|
+
connectErrorTime?: Date;
|
16
17
|
lastRequestEndTime?: Date;
|
18
|
+
attemptedRemoteAddresses?: string[];
|
19
|
+
connectProtocol?: string;
|
20
|
+
connectHost?: string;
|
21
|
+
connectPort?: string;
|
17
22
|
};
|
18
23
|
/**
|
19
24
|
* https://eggjs.org/en/core/httpclient.html#timing-boolean
|
@@ -1,6 +1,7 @@
|
|
1
1
|
import diagnosticsChannel from 'node:diagnostics_channel';
|
2
2
|
import { performance } from 'node:perf_hooks';
|
3
3
|
import { debuglog } from 'node:util';
|
4
|
+
import { Socket } from 'node:net';
|
4
5
|
import symbols from './symbols.js';
|
5
6
|
import { globalId, performanceTime } from './utils.js';
|
6
7
|
const debug = debuglog('urllib:DiagnosticsChannel');
|
@@ -32,8 +33,23 @@ function formatSocket(socket) {
|
|
32
33
|
localPort: socket[symbols.kSocketLocalPort],
|
33
34
|
remoteAddress: socket.remoteAddress,
|
34
35
|
remotePort: socket.remotePort,
|
36
|
+
attemptedAddresses: socket.autoSelectFamilyAttemptedAddresses,
|
37
|
+
connecting: socket.connecting,
|
35
38
|
};
|
36
39
|
}
|
40
|
+
// make sure error contains socket info
|
41
|
+
const kDestroy = Symbol('kDestroy');
|
42
|
+
Socket.prototype[kDestroy] = Socket.prototype.destroy;
|
43
|
+
Socket.prototype.destroy = function (err) {
|
44
|
+
if (err) {
|
45
|
+
Object.defineProperty(err, symbols.kErrorSocket, {
|
46
|
+
// don't show on console log
|
47
|
+
enumerable: false,
|
48
|
+
value: this,
|
49
|
+
});
|
50
|
+
}
|
51
|
+
return this[kDestroy](err);
|
52
|
+
};
|
37
53
|
export function initDiagnosticsChannel() {
|
38
54
|
// makre sure init global DiagnosticsChannel once
|
39
55
|
if (initedDiagnosticsChannel)
|
@@ -63,10 +79,34 @@ export function initDiagnosticsChannel() {
|
|
63
79
|
opaque[symbols.kRequestTiming].queuing = performanceTime(opaque[symbols.kRequestStartTime]);
|
64
80
|
});
|
65
81
|
// diagnosticsChannel.channel('undici:client:beforeConnect')
|
66
|
-
|
82
|
+
subscribe('undici:client:connectError', (message, name) => {
|
83
|
+
const { error, connectParams } = message;
|
84
|
+
let { socket } = message;
|
85
|
+
if (!socket && error[symbols.kErrorSocket]) {
|
86
|
+
socket = error[symbols.kErrorSocket];
|
87
|
+
}
|
88
|
+
if (socket) {
|
89
|
+
socket[symbols.kSocketId] = globalId('UndiciSocket');
|
90
|
+
socket[symbols.kSocketConnectErrorTime] = new Date();
|
91
|
+
socket[symbols.kHandledRequests] = 0;
|
92
|
+
socket[symbols.kHandledResponses] = 0;
|
93
|
+
// copy local address to symbol, avoid them be reset after request error throw
|
94
|
+
if (socket.localAddress) {
|
95
|
+
socket[symbols.kSocketLocalAddress] = socket.localAddress;
|
96
|
+
socket[symbols.kSocketLocalPort] = socket.localPort;
|
97
|
+
}
|
98
|
+
socket[symbols.kSocketConnectProtocol] = connectParams.protocol;
|
99
|
+
socket[symbols.kSocketConnectHost] = connectParams.host;
|
100
|
+
socket[symbols.kSocketConnectPort] = connectParams.port;
|
101
|
+
debug('[%s] Socket#%d connectError, connectParams: %o, error: %s, (sock: %o)', name, socket[symbols.kSocketId], connectParams, error.message, formatSocket(socket));
|
102
|
+
}
|
103
|
+
else {
|
104
|
+
debug('[%s] connectError, connectParams: %o, error: %o', name, connectParams, error);
|
105
|
+
}
|
106
|
+
});
|
67
107
|
// This message is published after a connection is established.
|
68
108
|
subscribe('undici:client:connected', (message, name) => {
|
69
|
-
const { socket } = message;
|
109
|
+
const { socket, connectParams } = message;
|
70
110
|
socket[symbols.kSocketId] = globalId('UndiciSocket');
|
71
111
|
socket[symbols.kSocketStartTime] = performance.now();
|
72
112
|
socket[symbols.kSocketConnectedTime] = new Date();
|
@@ -75,6 +115,9 @@ export function initDiagnosticsChannel() {
|
|
75
115
|
// copy local address to symbol, avoid them be reset after request error throw
|
76
116
|
socket[symbols.kSocketLocalAddress] = socket.localAddress;
|
77
117
|
socket[symbols.kSocketLocalPort] = socket.localPort;
|
118
|
+
socket[symbols.kSocketConnectProtocol] = connectParams.protocol;
|
119
|
+
socket[symbols.kSocketConnectHost] = connectParams.host;
|
120
|
+
socket[symbols.kSocketConnectPort] = connectParams.port;
|
78
121
|
debug('[%s] Socket#%d connected (sock: %o)', name, socket[symbols.kSocketId], formatSocket(socket));
|
79
122
|
});
|
80
123
|
// This message is published right before the first byte of the request is written to the socket.
|
package/dist/esm/index.d.ts
CHANGED
@@ -1,4 +1,6 @@
|
|
1
|
+
import { HttpClient } from './HttpClient.js';
|
1
2
|
import { RequestOptions, RequestURL } from './Request.js';
|
3
|
+
export declare function getDefaultHttpClient(): HttpClient;
|
2
4
|
export declare function request<T = any>(url: RequestURL, options?: RequestOptions): Promise<import("./Response.js").HttpClientResponse<T>>;
|
3
5
|
export declare function curl<T = any>(url: RequestURL, options?: RequestOptions): Promise<import("./Response.js").HttpClientResponse<T>>;
|
4
6
|
export { MockAgent, ProxyAgent, Agent, Dispatcher, setGlobalDispatcher, getGlobalDispatcher, } from 'undici';
|
@@ -6,6 +8,7 @@ export { HttpClient, HttpClient as HttpClient2, HEADER_USER_AGENT as USER_AGENT,
|
|
6
8
|
export { RequestOptions, RequestOptions as RequestOptions2, RequestURL, HttpMethod, FixJSONCtlCharsHandler, FixJSONCtlChars, } from './Request.js';
|
7
9
|
export { SocketInfo, Timing, RawResponseWithMeta, HttpClientResponse, } from './Response.js';
|
8
10
|
export { IncomingHttpHeaders, } from './IncomingHttpHeaders.js';
|
11
|
+
export * from './HttpClientError.js';
|
9
12
|
declare const _default: {
|
10
13
|
request: typeof request;
|
11
14
|
curl: typeof curl;
|
package/dist/esm/index.js
CHANGED
@@ -1,22 +1,25 @@
|
|
1
1
|
import LRU from 'ylru';
|
2
2
|
import { HttpClient, HEADER_USER_AGENT } from './HttpClient.js';
|
3
|
-
let
|
4
|
-
const
|
3
|
+
let httpClient;
|
4
|
+
const domainSocketHttpClients = new LRU(50);
|
5
|
+
export function getDefaultHttpClient() {
|
6
|
+
if (!httpClient) {
|
7
|
+
httpClient = new HttpClient();
|
8
|
+
}
|
9
|
+
return httpClient;
|
10
|
+
}
|
5
11
|
export async function request(url, options) {
|
6
12
|
if (options?.socketPath) {
|
7
|
-
let domainSocketHttpclient =
|
13
|
+
let domainSocketHttpclient = domainSocketHttpClients.get(options.socketPath);
|
8
14
|
if (!domainSocketHttpclient) {
|
9
15
|
domainSocketHttpclient = new HttpClient({
|
10
16
|
connect: { socketPath: options.socketPath },
|
11
17
|
});
|
12
|
-
|
18
|
+
domainSocketHttpClients.set(options.socketPath, domainSocketHttpclient);
|
13
19
|
}
|
14
20
|
return await domainSocketHttpclient.request(url, options);
|
15
21
|
}
|
16
|
-
|
17
|
-
httpclient = new HttpClient({});
|
18
|
-
}
|
19
|
-
return await httpclient.request(url, options);
|
22
|
+
return await getDefaultHttpClient().request(url, options);
|
20
23
|
}
|
21
24
|
// export curl method is keep compatible with urllib.curl()
|
22
25
|
// ```ts
|
@@ -27,8 +30,9 @@ export async function curl(url, options) {
|
|
27
30
|
return await request(url, options);
|
28
31
|
}
|
29
32
|
export { MockAgent, ProxyAgent, Agent, Dispatcher, setGlobalDispatcher, getGlobalDispatcher, } from 'undici';
|
30
|
-
// HttpClient2 is keep compatible with
|
33
|
+
// HttpClient2 is keep compatible with urllib@2 HttpClient2
|
31
34
|
export { HttpClient, HttpClient as HttpClient2, HEADER_USER_AGENT as USER_AGENT, } from './HttpClient.js';
|
35
|
+
export * from './HttpClientError.js';
|
32
36
|
export default {
|
33
37
|
request,
|
34
38
|
curl,
|
package/dist/esm/symbols.d.ts
CHANGED
@@ -2,9 +2,13 @@ declare const _default: {
|
|
2
2
|
kSocketId: symbol;
|
3
3
|
kSocketStartTime: symbol;
|
4
4
|
kSocketConnectedTime: symbol;
|
5
|
+
kSocketConnectErrorTime: symbol;
|
5
6
|
kSocketRequestEndTime: symbol;
|
6
7
|
kSocketLocalAddress: symbol;
|
7
8
|
kSocketLocalPort: symbol;
|
9
|
+
kSocketConnectHost: symbol;
|
10
|
+
kSocketConnectPort: symbol;
|
11
|
+
kSocketConnectProtocol: symbol;
|
8
12
|
kHandledRequests: symbol;
|
9
13
|
kHandledResponses: symbol;
|
10
14
|
kRequestSocket: symbol;
|
@@ -13,5 +17,6 @@ declare const _default: {
|
|
13
17
|
kEnableRequestTiming: symbol;
|
14
18
|
kRequestTiming: symbol;
|
15
19
|
kRequestOriginalOpaque: symbol;
|
20
|
+
kErrorSocket: symbol;
|
16
21
|
};
|
17
22
|
export default _default;
|
package/dist/esm/symbols.js
CHANGED
@@ -2,9 +2,13 @@ export default {
|
|
2
2
|
kSocketId: Symbol('socket id'),
|
3
3
|
kSocketStartTime: Symbol('socket start time'),
|
4
4
|
kSocketConnectedTime: Symbol('socket connected time'),
|
5
|
+
kSocketConnectErrorTime: Symbol('socket connectError time'),
|
5
6
|
kSocketRequestEndTime: Symbol('socket request end time'),
|
6
7
|
kSocketLocalAddress: Symbol('socket local address'),
|
7
8
|
kSocketLocalPort: Symbol('socket local port'),
|
9
|
+
kSocketConnectHost: Symbol('socket connect params: host'),
|
10
|
+
kSocketConnectPort: Symbol('socket connect params: port'),
|
11
|
+
kSocketConnectProtocol: Symbol('socket connect params: protocol'),
|
8
12
|
kHandledRequests: Symbol('handled requests per socket'),
|
9
13
|
kHandledResponses: Symbol('handled responses per socket'),
|
10
14
|
kRequestSocket: Symbol('request on the socket'),
|
@@ -13,4 +17,5 @@ export default {
|
|
13
17
|
kEnableRequestTiming: Symbol('enable request timing or not'),
|
14
18
|
kRequestTiming: Symbol('request timing'),
|
15
19
|
kRequestOriginalOpaque: Symbol('request original opaque'),
|
20
|
+
kErrorSocket: Symbol('socket of error'),
|
16
21
|
};
|
package/package.json
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
{
|
2
2
|
"name": "urllib",
|
3
|
-
"version": "3.
|
3
|
+
"version": "3.22.0",
|
4
4
|
"publishConfig": {
|
5
5
|
"tag": "latest"
|
6
6
|
},
|
@@ -32,8 +32,9 @@
|
|
32
32
|
"build:esm:test": "cd test/esm && rm -rf node_modules && npm link ../.. && node index.js",
|
33
33
|
"build:mts:test": "cd test/mts && rm -rf node_modules && npm link ../.. && tsc",
|
34
34
|
"build:test": "npm run build && npm run build:cjs:test && npm run build:esm:test && npm run build:mts:test && npm run test-tsc",
|
35
|
-
"test-tsc": "npm run test-tsc:cjs && npm run test-tsc:esm",
|
35
|
+
"test-tsc": "npm run test-tsc:cjs:es2021 && npm run test-tsc:cjs && npm run test-tsc:esm",
|
36
36
|
"test-tsc:cjs": "cd test/fixtures/ts && rm -rf node_modules && npm link ../../.. && npm run build",
|
37
|
+
"test-tsc:cjs:es2021": "cd test/fixtures/ts-cjs-es2021 && rm -rf node_modules && npm link ../../.. && npm run build",
|
37
38
|
"test-tsc:esm": "cd test/fixtures/ts-esm && rm -rf node_modules && npm link ../../.. && npm run build",
|
38
39
|
"test": "npm run lint && vitest run",
|
39
40
|
"test-keepalive": "cross-env TEST_KEEPALIVE_COUNT=50 vitest run --test-timeout 180000 keep-alive-header.test.ts",
|
@@ -68,7 +69,7 @@
|
|
68
69
|
"@types/qs": "^6.9.7",
|
69
70
|
"@types/selfsigned": "^2.0.1",
|
70
71
|
"@types/tar-stream": "^2.2.2",
|
71
|
-
"@vitest/coverage-v8": "
|
72
|
+
"@vitest/coverage-v8": "^1.0.1",
|
72
73
|
"busboy": "^1.6.0",
|
73
74
|
"cross-env": "^7.0.3",
|
74
75
|
"eslint": "^8.25.0",
|
@@ -81,7 +82,7 @@
|
|
81
82
|
"tshy": "^1.0.0",
|
82
83
|
"tshy-after": "^1.0.0",
|
83
84
|
"typescript": "^5.0.4",
|
84
|
-
"vitest": "
|
85
|
+
"vitest": "^1.0.1"
|
85
86
|
},
|
86
87
|
"engines": {
|
87
88
|
"node": ">= 14.19.3"
|
package/src/HttpAgent.ts
CHANGED
@@ -77,7 +77,7 @@ export class HttpAgent extends Agent {
|
|
77
77
|
}
|
78
78
|
const family = isIP(hostname);
|
79
79
|
if (family === 4 || family === 6) {
|
80
|
-
// if request hostname is ip, custom lookup won't
|
80
|
+
// if request hostname is ip, custom lookup won't execute
|
81
81
|
if (!this.#checkAddress(hostname, family)) {
|
82
82
|
throw new IllegalAddressError(hostname, hostname, family);
|
83
83
|
}
|
package/src/HttpClient.ts
CHANGED
@@ -22,7 +22,9 @@ import {
|
|
22
22
|
Dispatcher,
|
23
23
|
Agent,
|
24
24
|
getGlobalDispatcher,
|
25
|
+
Pool,
|
25
26
|
} from 'undici';
|
27
|
+
import undiciSymbols from 'undici/lib/core/symbols.js';
|
26
28
|
import { FormData as FormDataNode } from 'formdata-node';
|
27
29
|
import { FormDataEncoder } from 'form-data-encoder';
|
28
30
|
import createUserAgent from 'default-user-agent';
|
@@ -38,6 +40,7 @@ import { RawResponseWithMeta, HttpClientResponse, SocketInfo } from './Response.
|
|
38
40
|
import { parseJSON, sleep, digestAuthHeader, globalId, performanceTime, isReadable } from './utils.js';
|
39
41
|
import symbols from './symbols.js';
|
40
42
|
import { initDiagnosticsChannel } from './diagnosticsChannel.js';
|
43
|
+
import { HttpClientConnectTimeoutError, HttpClientRequestTimeoutError } from './HttpClientError.js';
|
41
44
|
|
42
45
|
type Exists<T> = T extends undefined ? never : T;
|
43
46
|
type UndiciRequestOption = Exists<Parameters<typeof undiciRequest>[1]>;
|
@@ -121,15 +124,6 @@ class BlobFromStream {
|
|
121
124
|
}
|
122
125
|
}
|
123
126
|
|
124
|
-
class HttpClientRequestTimeoutError extends Error {
|
125
|
-
constructor(timeout: number, options: ErrorOptions) {
|
126
|
-
const message = `Request timeout for ${timeout} ms`;
|
127
|
-
super(message, options);
|
128
|
-
this.name = this.constructor.name;
|
129
|
-
Error.captureStackTrace(this, this.constructor);
|
130
|
-
}
|
131
|
-
}
|
132
|
-
|
133
127
|
export const HEADER_USER_AGENT = createUserAgent('node-urllib', 'VERSION');
|
134
128
|
|
135
129
|
function getFileName(stream: Readable) {
|
@@ -144,7 +138,7 @@ function defaultIsRetry(response: HttpClientResponse) {
|
|
144
138
|
return response.status >= 500;
|
145
139
|
}
|
146
140
|
|
147
|
-
type RequestContext = {
|
141
|
+
export type RequestContext = {
|
148
142
|
retries: number;
|
149
143
|
socketErrorRetries: number;
|
150
144
|
requestStartTime?: number;
|
@@ -165,6 +159,20 @@ export type ResponseDiagnosticsMessage = {
|
|
165
159
|
error?: Error;
|
166
160
|
};
|
167
161
|
|
162
|
+
export interface PoolStat {
|
163
|
+
/** Number of open socket connections in this pool. */
|
164
|
+
connected: number;
|
165
|
+
/** Number of open socket connections in this pool that do not have an active request. */
|
166
|
+
free: number;
|
167
|
+
/** Number of pending requests across all clients in this pool. */
|
168
|
+
pending: number;
|
169
|
+
/** Number of queued requests across all clients in this pool. */
|
170
|
+
queued: number;
|
171
|
+
/** Number of currently active requests across all clients in this pool. */
|
172
|
+
running: number;
|
173
|
+
/** Number of active, pending, or queued requests across all clients in this pool. */
|
174
|
+
size: number;
|
175
|
+
}
|
168
176
|
|
169
177
|
export class HttpClient extends EventEmitter {
|
170
178
|
#defaultArgs?: RequestOptions;
|
@@ -195,11 +203,32 @@ export class HttpClient extends EventEmitter {
|
|
195
203
|
this.#dispatcher = dispatcher;
|
196
204
|
}
|
197
205
|
|
206
|
+
getDispatcherPoolStats() {
|
207
|
+
const agent = this.getDispatcher();
|
208
|
+
// origin => Pool Instance
|
209
|
+
const clients: Map<string, WeakRef<Pool>> = agent[undiciSymbols.kClients];
|
210
|
+
const poolStatsMap: Record<string, PoolStat> = {};
|
211
|
+
for (const [ key, ref ] of clients) {
|
212
|
+
const pool = ref.deref();
|
213
|
+
const stats = pool?.stats;
|
214
|
+
if (!stats) continue;
|
215
|
+
poolStatsMap[key] = {
|
216
|
+
connected: stats.connected,
|
217
|
+
free: stats.free,
|
218
|
+
pending: stats.pending,
|
219
|
+
queued: stats.queued,
|
220
|
+
running: stats.running,
|
221
|
+
size: stats.size,
|
222
|
+
} satisfies PoolStat;
|
223
|
+
}
|
224
|
+
return poolStatsMap;
|
225
|
+
}
|
226
|
+
|
198
227
|
async request<T = any>(url: RequestURL, options?: RequestOptions) {
|
199
228
|
return await this.#requestInternal<T>(url, options);
|
200
229
|
}
|
201
230
|
|
202
|
-
// alias to request, keep compatible with
|
231
|
+
// alias to request, keep compatible with urllib@2 HttpClient.curl
|
203
232
|
async curl<T = any>(url: RequestURL, options?: RequestOptions) {
|
204
233
|
return await this.request<T>(url, options);
|
205
234
|
}
|
@@ -653,13 +682,15 @@ export class HttpClient extends EventEmitter {
|
|
653
682
|
}
|
654
683
|
|
655
684
|
return clientResponse;
|
656
|
-
} catch (
|
657
|
-
debug('Request#%d throw error: %s', requestId,
|
658
|
-
let err =
|
685
|
+
} catch (rawError: any) {
|
686
|
+
debug('Request#%d throw error: %s', requestId, rawError);
|
687
|
+
let err = rawError;
|
659
688
|
if (err.name === 'HeadersTimeoutError') {
|
660
|
-
err = new HttpClientRequestTimeoutError(headersTimeout, { cause:
|
689
|
+
err = new HttpClientRequestTimeoutError(headersTimeout, { cause: err });
|
661
690
|
} else if (err.name === 'BodyTimeoutError') {
|
662
|
-
err = new HttpClientRequestTimeoutError(bodyTimeout, { cause:
|
691
|
+
err = new HttpClientRequestTimeoutError(bodyTimeout, { cause: err });
|
692
|
+
} else if (err.code === 'UND_ERR_CONNECT_TIMEOUT') {
|
693
|
+
err = new HttpClientConnectTimeoutError(err.message, err.code, { cause: err });
|
663
694
|
} else if (err.code === 'UND_ERR_SOCKET' || err.code === 'ECONNRESET') {
|
664
695
|
// auto retry on socket error, https://github.com/node-modules/urllib/issues/454
|
665
696
|
if (args.socketErrorRetry > 0 && requestContext.socketErrorRetries < args.socketErrorRetry) {
|
@@ -681,7 +712,7 @@ export class HttpClient extends EventEmitter {
|
|
681
712
|
res.requestUrls.push(requestUrl.href);
|
682
713
|
}
|
683
714
|
res.rt = performanceTime(requestStartTime);
|
684
|
-
this.#updateSocketInfo(socketInfo, internalOpaque);
|
715
|
+
this.#updateSocketInfo(socketInfo, internalOpaque, rawError);
|
685
716
|
|
686
717
|
channels.response.publish({
|
687
718
|
request: reqMeta,
|
@@ -704,21 +735,38 @@ export class HttpClient extends EventEmitter {
|
|
704
735
|
}
|
705
736
|
}
|
706
737
|
|
707
|
-
#updateSocketInfo(socketInfo: SocketInfo, internalOpaque: any) {
|
708
|
-
const socket = internalOpaque[symbols.kRequestSocket];
|
738
|
+
#updateSocketInfo(socketInfo: SocketInfo, internalOpaque: any, err?: any) {
|
739
|
+
const socket = internalOpaque[symbols.kRequestSocket] ?? err?.[symbols.kErrorSocket];
|
709
740
|
if (socket) {
|
710
741
|
socketInfo.id = socket[symbols.kSocketId];
|
711
742
|
socketInfo.handledRequests = socket[symbols.kHandledRequests];
|
712
743
|
socketInfo.handledResponses = socket[symbols.kHandledResponses];
|
713
|
-
|
714
|
-
|
715
|
-
|
716
|
-
|
717
|
-
|
744
|
+
if (socket[symbols.kSocketLocalAddress]) {
|
745
|
+
socketInfo.localAddress = socket[symbols.kSocketLocalAddress];
|
746
|
+
socketInfo.localPort = socket[symbols.kSocketLocalPort];
|
747
|
+
}
|
748
|
+
if (socket.remoteAddress) {
|
749
|
+
socketInfo.remoteAddress = socket.remoteAddress;
|
750
|
+
socketInfo.remotePort = socket.remotePort;
|
751
|
+
socketInfo.remoteFamily = socket.remoteFamily;
|
752
|
+
}
|
718
753
|
socketInfo.bytesRead = socket.bytesRead;
|
719
754
|
socketInfo.bytesWritten = socket.bytesWritten;
|
720
|
-
|
721
|
-
|
755
|
+
if (socket[symbols.kSocketConnectErrorTime]) {
|
756
|
+
socketInfo.connectErrorTime = socket[symbols.kSocketConnectErrorTime];
|
757
|
+
if (Array.isArray(socket.autoSelectFamilyAttemptedAddresses)) {
|
758
|
+
socketInfo.attemptedRemoteAddresses = socket.autoSelectFamilyAttemptedAddresses;
|
759
|
+
}
|
760
|
+
socketInfo.connectProtocol = socket[symbols.kSocketConnectProtocol];
|
761
|
+
socketInfo.connectHost = socket[symbols.kSocketConnectHost];
|
762
|
+
socketInfo.connectPort = socket[symbols.kSocketConnectPort];
|
763
|
+
}
|
764
|
+
if (socket[symbols.kSocketConnectedTime]) {
|
765
|
+
socketInfo.connectedTime = socket[symbols.kSocketConnectedTime];
|
766
|
+
}
|
767
|
+
if (socket[symbols.kSocketRequestEndTime]) {
|
768
|
+
socketInfo.lastRequestEndTime = socket[symbols.kSocketRequestEndTime];
|
769
|
+
}
|
722
770
|
socket[symbols.kSocketRequestEndTime] = new Date();
|
723
771
|
}
|
724
772
|
}
|
@@ -0,0 +1,34 @@
|
|
1
|
+
import type { RawResponseWithMeta, SocketInfo } from './Response.js';
|
2
|
+
import type { IncomingHttpHeaders } from './IncomingHttpHeaders.js';
|
3
|
+
|
4
|
+
// need to support ES2021
|
5
|
+
interface ErrorOptions {
|
6
|
+
cause?: Error;
|
7
|
+
}
|
8
|
+
|
9
|
+
export class HttpClientRequestError extends Error {
|
10
|
+
status?: number;
|
11
|
+
headers?: IncomingHttpHeaders;
|
12
|
+
socket?: SocketInfo;
|
13
|
+
res?: RawResponseWithMeta;
|
14
|
+
}
|
15
|
+
|
16
|
+
export class HttpClientRequestTimeoutError extends HttpClientRequestError {
|
17
|
+
constructor(timeout: number, options: ErrorOptions) {
|
18
|
+
const message = `Request timeout for ${timeout} ms`;
|
19
|
+
super(message, options);
|
20
|
+
this.name = this.constructor.name;
|
21
|
+
Error.captureStackTrace(this, this.constructor);
|
22
|
+
}
|
23
|
+
}
|
24
|
+
|
25
|
+
export class HttpClientConnectTimeoutError extends HttpClientRequestError {
|
26
|
+
code: string;
|
27
|
+
|
28
|
+
constructor(message: string, code: string, options: ErrorOptions) {
|
29
|
+
super(message, options);
|
30
|
+
this.name = this.constructor.name;
|
31
|
+
this.code = code;
|
32
|
+
Error.captureStackTrace(this, this.constructor);
|
33
|
+
}
|
34
|
+
}
|
package/src/Response.ts
CHANGED
@@ -13,7 +13,12 @@ export type SocketInfo = {
|
|
13
13
|
handledRequests: number;
|
14
14
|
handledResponses: number;
|
15
15
|
connectedTime?: Date;
|
16
|
+
connectErrorTime?: Date;
|
16
17
|
lastRequestEndTime?: Date;
|
18
|
+
attemptedRemoteAddresses?: string[];
|
19
|
+
connectProtocol?: string;
|
20
|
+
connectHost?: string;
|
21
|
+
connectPort?: string;
|
17
22
|
};
|
18
23
|
|
19
24
|
/**
|