urllib 3.19.3 → 3.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -8
- package/dist/commonjs/HttpAgent.js +1 -1
- package/dist/commonjs/HttpClient.d.ts +7 -0
- package/dist/commonjs/HttpClient.js +49 -25
- package/dist/commonjs/HttpClientError.d.ts +19 -0
- package/dist/commonjs/HttpClientError.js +29 -0
- package/dist/commonjs/Response.d.ts +5 -0
- package/dist/commonjs/diagnosticsChannel.js +45 -2
- package/dist/commonjs/index.d.ts +1 -0
- package/dist/commonjs/index.js +16 -1
- package/dist/commonjs/package.json +3 -1
- package/dist/commonjs/symbols.d.ts +5 -0
- package/dist/commonjs/symbols.js +5 -0
- package/dist/commonjs/utils.js +6 -6
- package/dist/esm/HttpAgent.js +1 -1
- package/dist/esm/HttpClient.d.ts +7 -0
- package/dist/esm/HttpClient.js +50 -26
- package/dist/esm/HttpClientError.d.ts +19 -0
- package/dist/esm/HttpClientError.js +23 -0
- package/dist/esm/Response.d.ts +5 -0
- package/dist/esm/diagnosticsChannel.js +45 -2
- package/dist/esm/index.d.ts +1 -0
- package/dist/esm/index.js +2 -1
- package/dist/esm/package.json +3 -1
- package/dist/esm/symbols.d.ts +5 -0
- package/dist/esm/symbols.js +5 -0
- package/dist/esm/utils.js +6 -6
- package/package.json +33 -30
- package/src/HttpAgent.ts +1 -1
- package/src/HttpClient.ts +54 -26
- package/src/HttpClientError.ts +34 -0
- package/src/Response.ts +5 -0
- package/src/diagnosticsChannel.ts +48 -2
- package/src/index.ts +3 -2
- package/src/symbols.ts +5 -0
package/dist/esm/HttpClient.js
CHANGED
@@ -10,7 +10,7 @@ import { basename } from 'node:path';
|
|
10
10
|
import { createReadStream } from 'node:fs';
|
11
11
|
import { format as urlFormat } from 'node:url';
|
12
12
|
import { performance } from 'node:perf_hooks';
|
13
|
-
import { FormData as FormDataNative, request as undiciRequest, } from 'undici';
|
13
|
+
import { FormData as FormDataNative, request as undiciRequest, Agent, getGlobalDispatcher, } from 'undici';
|
14
14
|
import { FormData as FormDataNode } from 'formdata-node';
|
15
15
|
import { FormDataEncoder } from 'form-data-encoder';
|
16
16
|
import createUserAgent from 'default-user-agent';
|
@@ -23,6 +23,7 @@ import { HttpAgent } from './HttpAgent.js';
|
|
23
23
|
import { parseJSON, sleep, digestAuthHeader, globalId, performanceTime, isReadable } from './utils.js';
|
24
24
|
import symbols from './symbols.js';
|
25
25
|
import { initDiagnosticsChannel } from './diagnosticsChannel.js';
|
26
|
+
import { HttpClientConnectTimeoutError, HttpClientRequestTimeoutError } from './HttpClientError.js';
|
26
27
|
const PROTO_RE = /^https?:\/\//i;
|
27
28
|
const FormData = FormDataNative ?? FormDataNode;
|
28
29
|
// impl promise pipeline on Node.js 14
|
@@ -59,15 +60,7 @@ class BlobFromStream {
|
|
59
60
|
return 'Blob';
|
60
61
|
}
|
61
62
|
}
|
62
|
-
|
63
|
-
constructor(timeout, options) {
|
64
|
-
const message = `Request timeout for ${timeout} ms`;
|
65
|
-
super(message, options);
|
66
|
-
this.name = this.constructor.name;
|
67
|
-
Error.captureStackTrace(this, this.constructor);
|
68
|
-
}
|
69
|
-
}
|
70
|
-
export const HEADER_USER_AGENT = createUserAgent('node-urllib', '3.19.3');
|
63
|
+
export const HEADER_USER_AGENT = createUserAgent('node-urllib', '3.21.0');
|
71
64
|
function getFileName(stream) {
|
72
65
|
const filePath = stream.path;
|
73
66
|
if (filePath) {
|
@@ -88,15 +81,26 @@ export class HttpClient extends EventEmitter {
|
|
88
81
|
constructor(clientOptions) {
|
89
82
|
super();
|
90
83
|
this.#defaultArgs = clientOptions?.defaultArgs;
|
91
|
-
if (clientOptions?.lookup || clientOptions?.checkAddress
|
84
|
+
if (clientOptions?.lookup || clientOptions?.checkAddress) {
|
92
85
|
this.#dispatcher = new HttpAgent({
|
93
86
|
lookup: clientOptions.lookup,
|
94
87
|
checkAddress: clientOptions.checkAddress,
|
95
88
|
connect: clientOptions.connect,
|
96
89
|
});
|
97
90
|
}
|
91
|
+
else if (clientOptions?.connect) {
|
92
|
+
this.#dispatcher = new Agent({
|
93
|
+
connect: clientOptions.connect,
|
94
|
+
});
|
95
|
+
}
|
98
96
|
initDiagnosticsChannel();
|
99
97
|
}
|
98
|
+
getDispatcher() {
|
99
|
+
return this.#dispatcher ?? getGlobalDispatcher();
|
100
|
+
}
|
101
|
+
setDispatcher(dispatcher) {
|
102
|
+
this.#dispatcher = dispatcher;
|
103
|
+
}
|
100
104
|
async request(url, options) {
|
101
105
|
return await this.#requestInternal(url, options);
|
102
106
|
}
|
@@ -563,14 +567,17 @@ export class HttpClient extends EventEmitter {
|
|
563
567
|
}
|
564
568
|
return clientResponse;
|
565
569
|
}
|
566
|
-
catch (
|
567
|
-
debug('Request#%d throw error: %s', requestId,
|
568
|
-
let err =
|
570
|
+
catch (rawError) {
|
571
|
+
debug('Request#%d throw error: %s', requestId, rawError);
|
572
|
+
let err = rawError;
|
569
573
|
if (err.name === 'HeadersTimeoutError') {
|
570
|
-
err = new HttpClientRequestTimeoutError(headersTimeout, { cause:
|
574
|
+
err = new HttpClientRequestTimeoutError(headersTimeout, { cause: err });
|
571
575
|
}
|
572
576
|
else if (err.name === 'BodyTimeoutError') {
|
573
|
-
err = new HttpClientRequestTimeoutError(bodyTimeout, { cause:
|
577
|
+
err = new HttpClientRequestTimeoutError(bodyTimeout, { cause: err });
|
578
|
+
}
|
579
|
+
else if (err.code === 'UND_ERR_CONNECT_TIMEOUT') {
|
580
|
+
err = new HttpClientConnectTimeoutError(err.message, err.code, { cause: err });
|
574
581
|
}
|
575
582
|
else if (err.code === 'UND_ERR_SOCKET' || err.code === 'ECONNRESET') {
|
576
583
|
// auto retry on socket error, https://github.com/node-modules/urllib/issues/454
|
@@ -593,7 +600,7 @@ export class HttpClient extends EventEmitter {
|
|
593
600
|
res.requestUrls.push(requestUrl.href);
|
594
601
|
}
|
595
602
|
res.rt = performanceTime(requestStartTime);
|
596
|
-
this.#updateSocketInfo(socketInfo, internalOpaque);
|
603
|
+
this.#updateSocketInfo(socketInfo, internalOpaque, rawError);
|
597
604
|
channels.response.publish({
|
598
605
|
request: reqMeta,
|
599
606
|
response: res,
|
@@ -614,21 +621,38 @@ export class HttpClient extends EventEmitter {
|
|
614
621
|
throw err;
|
615
622
|
}
|
616
623
|
}
|
617
|
-
#updateSocketInfo(socketInfo, internalOpaque) {
|
618
|
-
const socket = internalOpaque[symbols.kRequestSocket];
|
624
|
+
#updateSocketInfo(socketInfo, internalOpaque, err) {
|
625
|
+
const socket = internalOpaque[symbols.kRequestSocket] ?? err?.[symbols.kErrorSocket];
|
619
626
|
if (socket) {
|
620
627
|
socketInfo.id = socket[symbols.kSocketId];
|
621
628
|
socketInfo.handledRequests = socket[symbols.kHandledRequests];
|
622
629
|
socketInfo.handledResponses = socket[symbols.kHandledResponses];
|
623
|
-
|
624
|
-
|
625
|
-
|
626
|
-
|
627
|
-
|
630
|
+
if (socket[symbols.kSocketLocalAddress]) {
|
631
|
+
socketInfo.localAddress = socket[symbols.kSocketLocalAddress];
|
632
|
+
socketInfo.localPort = socket[symbols.kSocketLocalPort];
|
633
|
+
}
|
634
|
+
if (socket.remoteAddress) {
|
635
|
+
socketInfo.remoteAddress = socket.remoteAddress;
|
636
|
+
socketInfo.remotePort = socket.remotePort;
|
637
|
+
socketInfo.remoteFamily = socket.remoteFamily;
|
638
|
+
}
|
628
639
|
socketInfo.bytesRead = socket.bytesRead;
|
629
640
|
socketInfo.bytesWritten = socket.bytesWritten;
|
630
|
-
|
631
|
-
|
641
|
+
if (socket[symbols.kSocketConnectErrorTime]) {
|
642
|
+
socketInfo.connectErrorTime = socket[symbols.kSocketConnectErrorTime];
|
643
|
+
if (Array.isArray(socket.autoSelectFamilyAttemptedAddresses)) {
|
644
|
+
socketInfo.attemptedRemoteAddresses = socket.autoSelectFamilyAttemptedAddresses;
|
645
|
+
}
|
646
|
+
socketInfo.connectProtocol = socket[symbols.kSocketConnectProtocol];
|
647
|
+
socketInfo.connectHost = socket[symbols.kSocketConnectHost];
|
648
|
+
socketInfo.connectPort = socket[symbols.kSocketConnectPort];
|
649
|
+
}
|
650
|
+
if (socket[symbols.kSocketConnectedTime]) {
|
651
|
+
socketInfo.connectedTime = socket[symbols.kSocketConnectedTime];
|
652
|
+
}
|
653
|
+
if (socket[symbols.kSocketRequestEndTime]) {
|
654
|
+
socketInfo.lastRequestEndTime = socket[symbols.kSocketRequestEndTime];
|
655
|
+
}
|
632
656
|
socket[symbols.kSocketRequestEndTime] = new Date();
|
633
657
|
}
|
634
658
|
}
|
@@ -0,0 +1,19 @@
|
|
1
|
+
import type { RawResponseWithMeta, SocketInfo } from './Response.js';
|
2
|
+
import type { IncomingHttpHeaders } from './IncomingHttpHeaders.js';
|
3
|
+
interface ErrorOptions {
|
4
|
+
cause?: Error;
|
5
|
+
}
|
6
|
+
export declare class HttpClientRequestError extends Error {
|
7
|
+
status?: number;
|
8
|
+
headers?: IncomingHttpHeaders;
|
9
|
+
socket?: SocketInfo;
|
10
|
+
res?: RawResponseWithMeta;
|
11
|
+
}
|
12
|
+
export declare class HttpClientRequestTimeoutError extends HttpClientRequestError {
|
13
|
+
constructor(timeout: number, options: ErrorOptions);
|
14
|
+
}
|
15
|
+
export declare class HttpClientConnectTimeoutError extends HttpClientRequestError {
|
16
|
+
code: string;
|
17
|
+
constructor(message: string, code: string, options: ErrorOptions);
|
18
|
+
}
|
19
|
+
export {};
|
@@ -0,0 +1,23 @@
|
|
1
|
+
export class HttpClientRequestError extends Error {
|
2
|
+
status;
|
3
|
+
headers;
|
4
|
+
socket;
|
5
|
+
res;
|
6
|
+
}
|
7
|
+
export class HttpClientRequestTimeoutError extends HttpClientRequestError {
|
8
|
+
constructor(timeout, options) {
|
9
|
+
const message = `Request timeout for ${timeout} ms`;
|
10
|
+
super(message, options);
|
11
|
+
this.name = this.constructor.name;
|
12
|
+
Error.captureStackTrace(this, this.constructor);
|
13
|
+
}
|
14
|
+
}
|
15
|
+
export class HttpClientConnectTimeoutError extends HttpClientRequestError {
|
16
|
+
code;
|
17
|
+
constructor(message, code, options) {
|
18
|
+
super(message, options);
|
19
|
+
this.name = this.constructor.name;
|
20
|
+
this.code = code;
|
21
|
+
Error.captureStackTrace(this, this.constructor);
|
22
|
+
}
|
23
|
+
}
|
package/dist/esm/Response.d.ts
CHANGED
@@ -13,7 +13,12 @@ export type SocketInfo = {
|
|
13
13
|
handledRequests: number;
|
14
14
|
handledResponses: number;
|
15
15
|
connectedTime?: Date;
|
16
|
+
connectErrorTime?: Date;
|
16
17
|
lastRequestEndTime?: Date;
|
18
|
+
attemptedRemoteAddresses?: string[];
|
19
|
+
connectProtocol?: string;
|
20
|
+
connectHost?: string;
|
21
|
+
connectPort?: string;
|
17
22
|
};
|
18
23
|
/**
|
19
24
|
* https://eggjs.org/en/core/httpclient.html#timing-boolean
|
@@ -1,6 +1,7 @@
|
|
1
1
|
import diagnosticsChannel from 'node:diagnostics_channel';
|
2
2
|
import { performance } from 'node:perf_hooks';
|
3
3
|
import { debuglog } from 'node:util';
|
4
|
+
import { Socket } from 'node:net';
|
4
5
|
import symbols from './symbols.js';
|
5
6
|
import { globalId, performanceTime } from './utils.js';
|
6
7
|
const debug = debuglog('urllib:DiagnosticsChannel');
|
@@ -32,8 +33,23 @@ function formatSocket(socket) {
|
|
32
33
|
localPort: socket[symbols.kSocketLocalPort],
|
33
34
|
remoteAddress: socket.remoteAddress,
|
34
35
|
remotePort: socket.remotePort,
|
36
|
+
attemptedAddresses: socket.autoSelectFamilyAttemptedAddresses,
|
37
|
+
connecting: socket.connecting,
|
35
38
|
};
|
36
39
|
}
|
40
|
+
// make sure error contains socket info
|
41
|
+
const kDestroy = Symbol('kDestroy');
|
42
|
+
Socket.prototype[kDestroy] = Socket.prototype.destroy;
|
43
|
+
Socket.prototype.destroy = function (err) {
|
44
|
+
if (err) {
|
45
|
+
Object.defineProperty(err, symbols.kErrorSocket, {
|
46
|
+
// don't show on console log
|
47
|
+
enumerable: false,
|
48
|
+
value: this,
|
49
|
+
});
|
50
|
+
}
|
51
|
+
return this[kDestroy](err);
|
52
|
+
};
|
37
53
|
export function initDiagnosticsChannel() {
|
38
54
|
// makre sure init global DiagnosticsChannel once
|
39
55
|
if (initedDiagnosticsChannel)
|
@@ -63,10 +79,34 @@ export function initDiagnosticsChannel() {
|
|
63
79
|
opaque[symbols.kRequestTiming].queuing = performanceTime(opaque[symbols.kRequestStartTime]);
|
64
80
|
});
|
65
81
|
// diagnosticsChannel.channel('undici:client:beforeConnect')
|
66
|
-
|
82
|
+
subscribe('undici:client:connectError', (message, name) => {
|
83
|
+
const { error, connectParams } = message;
|
84
|
+
let { socket } = message;
|
85
|
+
if (!socket && error[symbols.kErrorSocket]) {
|
86
|
+
socket = error[symbols.kErrorSocket];
|
87
|
+
}
|
88
|
+
if (socket) {
|
89
|
+
socket[symbols.kSocketId] = globalId('UndiciSocket');
|
90
|
+
socket[symbols.kSocketConnectErrorTime] = new Date();
|
91
|
+
socket[symbols.kHandledRequests] = 0;
|
92
|
+
socket[symbols.kHandledResponses] = 0;
|
93
|
+
// copy local address to symbol, avoid them be reset after request error throw
|
94
|
+
if (socket.localAddress) {
|
95
|
+
socket[symbols.kSocketLocalAddress] = socket.localAddress;
|
96
|
+
socket[symbols.kSocketLocalPort] = socket.localPort;
|
97
|
+
}
|
98
|
+
socket[symbols.kSocketConnectProtocol] = connectParams.protocol;
|
99
|
+
socket[symbols.kSocketConnectHost] = connectParams.host;
|
100
|
+
socket[symbols.kSocketConnectPort] = connectParams.port;
|
101
|
+
debug('[%s] Socket#%d connectError, connectParams: %o, error: %s, (sock: %o)', name, socket[symbols.kSocketId], connectParams, error.message, formatSocket(socket));
|
102
|
+
}
|
103
|
+
else {
|
104
|
+
debug('[%s] connectError, connectParams: %o, error: %o', name, connectParams, error);
|
105
|
+
}
|
106
|
+
});
|
67
107
|
// This message is published after a connection is established.
|
68
108
|
subscribe('undici:client:connected', (message, name) => {
|
69
|
-
const { socket } = message;
|
109
|
+
const { socket, connectParams } = message;
|
70
110
|
socket[symbols.kSocketId] = globalId('UndiciSocket');
|
71
111
|
socket[symbols.kSocketStartTime] = performance.now();
|
72
112
|
socket[symbols.kSocketConnectedTime] = new Date();
|
@@ -75,6 +115,9 @@ export function initDiagnosticsChannel() {
|
|
75
115
|
// copy local address to symbol, avoid them be reset after request error throw
|
76
116
|
socket[symbols.kSocketLocalAddress] = socket.localAddress;
|
77
117
|
socket[symbols.kSocketLocalPort] = socket.localPort;
|
118
|
+
socket[symbols.kSocketConnectProtocol] = connectParams.protocol;
|
119
|
+
socket[symbols.kSocketConnectHost] = connectParams.host;
|
120
|
+
socket[symbols.kSocketConnectPort] = connectParams.port;
|
78
121
|
debug('[%s] Socket#%d connected (sock: %o)', name, socket[symbols.kSocketId], formatSocket(socket));
|
79
122
|
});
|
80
123
|
// This message is published right before the first byte of the request is written to the socket.
|
package/dist/esm/index.d.ts
CHANGED
@@ -6,6 +6,7 @@ export { HttpClient, HttpClient as HttpClient2, HEADER_USER_AGENT as USER_AGENT,
|
|
6
6
|
export { RequestOptions, RequestOptions as RequestOptions2, RequestURL, HttpMethod, FixJSONCtlCharsHandler, FixJSONCtlChars, } from './Request.js';
|
7
7
|
export { SocketInfo, Timing, RawResponseWithMeta, HttpClientResponse, } from './Response.js';
|
8
8
|
export { IncomingHttpHeaders, } from './IncomingHttpHeaders.js';
|
9
|
+
export * from './HttpClientError.js';
|
9
10
|
declare const _default: {
|
10
11
|
request: typeof request;
|
11
12
|
curl: typeof curl;
|
package/dist/esm/index.js
CHANGED
@@ -18,7 +18,7 @@ export async function request(url, options) {
|
|
18
18
|
}
|
19
19
|
return await httpclient.request(url, options);
|
20
20
|
}
|
21
|
-
// export curl method is keep compatible with
|
21
|
+
// export curl method is keep compatible with urllib.curl()
|
22
22
|
// ```ts
|
23
23
|
// import * as urllib from 'urllib';
|
24
24
|
// urllib.curl(url);
|
@@ -29,6 +29,7 @@ export async function curl(url, options) {
|
|
29
29
|
export { MockAgent, ProxyAgent, Agent, Dispatcher, setGlobalDispatcher, getGlobalDispatcher, } from 'undici';
|
30
30
|
// HttpClient2 is keep compatible with urlib@2 HttpClient2
|
31
31
|
export { HttpClient, HttpClient as HttpClient2, HEADER_USER_AGENT as USER_AGENT, } from './HttpClient.js';
|
32
|
+
export * from './HttpClientError.js';
|
32
33
|
export default {
|
33
34
|
request,
|
34
35
|
curl,
|
package/dist/esm/package.json
CHANGED
package/dist/esm/symbols.d.ts
CHANGED
@@ -2,9 +2,13 @@ declare const _default: {
|
|
2
2
|
kSocketId: symbol;
|
3
3
|
kSocketStartTime: symbol;
|
4
4
|
kSocketConnectedTime: symbol;
|
5
|
+
kSocketConnectErrorTime: symbol;
|
5
6
|
kSocketRequestEndTime: symbol;
|
6
7
|
kSocketLocalAddress: symbol;
|
7
8
|
kSocketLocalPort: symbol;
|
9
|
+
kSocketConnectHost: symbol;
|
10
|
+
kSocketConnectPort: symbol;
|
11
|
+
kSocketConnectProtocol: symbol;
|
8
12
|
kHandledRequests: symbol;
|
9
13
|
kHandledResponses: symbol;
|
10
14
|
kRequestSocket: symbol;
|
@@ -13,5 +17,6 @@ declare const _default: {
|
|
13
17
|
kEnableRequestTiming: symbol;
|
14
18
|
kRequestTiming: symbol;
|
15
19
|
kRequestOriginalOpaque: symbol;
|
20
|
+
kErrorSocket: symbol;
|
16
21
|
};
|
17
22
|
export default _default;
|
package/dist/esm/symbols.js
CHANGED
@@ -2,9 +2,13 @@ export default {
|
|
2
2
|
kSocketId: Symbol('socket id'),
|
3
3
|
kSocketStartTime: Symbol('socket start time'),
|
4
4
|
kSocketConnectedTime: Symbol('socket connected time'),
|
5
|
+
kSocketConnectErrorTime: Symbol('socket connectError time'),
|
5
6
|
kSocketRequestEndTime: Symbol('socket request end time'),
|
6
7
|
kSocketLocalAddress: Symbol('socket local address'),
|
7
8
|
kSocketLocalPort: Symbol('socket local port'),
|
9
|
+
kSocketConnectHost: Symbol('socket connect params: host'),
|
10
|
+
kSocketConnectPort: Symbol('socket connect params: port'),
|
11
|
+
kSocketConnectProtocol: Symbol('socket connect params: protocol'),
|
8
12
|
kHandledRequests: Symbol('handled requests per socket'),
|
9
13
|
kHandledResponses: Symbol('handled responses per socket'),
|
10
14
|
kRequestSocket: Symbol('request on the socket'),
|
@@ -13,4 +17,5 @@ export default {
|
|
13
17
|
kEnableRequestTiming: Symbol('enable request timing or not'),
|
14
18
|
kRequestTiming: Symbol('request timing'),
|
15
19
|
kRequestOriginalOpaque: Symbol('request original opaque'),
|
20
|
+
kErrorSocket: Symbol('socket of error'),
|
16
21
|
};
|
package/dist/esm/utils.js
CHANGED
@@ -2,12 +2,12 @@ import { randomBytes, createHash } from 'node:crypto';
|
|
2
2
|
import { Readable } from 'node:stream';
|
3
3
|
import { performance } from 'node:perf_hooks';
|
4
4
|
const JSONCtlCharsMap = {
|
5
|
-
'"': '\\"',
|
6
|
-
'\\': '\\\\',
|
7
|
-
'\b': '\\b',
|
8
|
-
'\f': '\\f',
|
9
|
-
'\n': '\\n',
|
10
|
-
'\r': '\\r',
|
5
|
+
'"': '\\"', // \u0022
|
6
|
+
'\\': '\\\\', // \u005c
|
7
|
+
'\b': '\\b', // \u0008
|
8
|
+
'\f': '\\f', // \u000c
|
9
|
+
'\n': '\\n', // \u000a
|
10
|
+
'\r': '\\r', // \u000d
|
11
11
|
'\t': '\\t', // \u0009
|
12
12
|
};
|
13
13
|
/* eslint no-control-regex: "off"*/
|
package/package.json
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
{
|
2
2
|
"name": "urllib",
|
3
|
-
"version": "3.
|
3
|
+
"version": "3.21.0",
|
4
4
|
"publishConfig": {
|
5
5
|
"tag": "latest"
|
6
6
|
},
|
@@ -18,30 +18,6 @@
|
|
18
18
|
],
|
19
19
|
"author": "fengmk2 <fengmk2@gmail.com> (https://github.com/fengmk2)",
|
20
20
|
"homepage": "https://github.com/node-modules/urllib",
|
21
|
-
"type": "module",
|
22
|
-
"tshy": {
|
23
|
-
"exports": {
|
24
|
-
".": "./src/index.ts",
|
25
|
-
"./package.json": "./package.json"
|
26
|
-
}
|
27
|
-
},
|
28
|
-
"exports": {
|
29
|
-
".": {
|
30
|
-
"import": {
|
31
|
-
"types": "./dist/esm/index.d.ts",
|
32
|
-
"default": "./dist/esm/index.js"
|
33
|
-
},
|
34
|
-
"require": {
|
35
|
-
"types": "./dist/commonjs/index.d.ts",
|
36
|
-
"default": "./dist/commonjs/index.js"
|
37
|
-
}
|
38
|
-
},
|
39
|
-
"./package.json": "./package.json"
|
40
|
-
},
|
41
|
-
"files": [
|
42
|
-
"dist",
|
43
|
-
"src"
|
44
|
-
],
|
45
21
|
"repository": {
|
46
22
|
"type": "git",
|
47
23
|
"url": "git://github.com/node-modules/urllib.git"
|
@@ -56,12 +32,14 @@
|
|
56
32
|
"build:esm:test": "cd test/esm && rm -rf node_modules && npm link ../.. && node index.js",
|
57
33
|
"build:mts:test": "cd test/mts && rm -rf node_modules && npm link ../.. && tsc",
|
58
34
|
"build:test": "npm run build && npm run build:cjs:test && npm run build:esm:test && npm run build:mts:test && npm run test-tsc",
|
59
|
-
"test-tsc": "npm run test-tsc:cjs && npm run test-tsc:esm",
|
35
|
+
"test-tsc": "npm run test-tsc:cjs:es2021 && npm run test-tsc:cjs && npm run test-tsc:esm",
|
60
36
|
"test-tsc:cjs": "cd test/fixtures/ts && rm -rf node_modules && npm link ../../.. && npm run build",
|
37
|
+
"test-tsc:cjs:es2021": "cd test/fixtures/ts-cjs-es2021 && rm -rf node_modules && npm link ../../.. && npm run build",
|
61
38
|
"test-tsc:esm": "cd test/fixtures/ts-esm && rm -rf node_modules && npm link ../../.. && npm run build",
|
62
39
|
"test": "npm run lint && vitest run",
|
63
40
|
"test-keepalive": "cross-env TEST_KEEPALIVE_COUNT=50 vitest run --test-timeout 180000 keep-alive-header.test.ts",
|
64
41
|
"cov": "vitest run --coverage",
|
42
|
+
"preci": "node scripts/pre_test.js",
|
65
43
|
"ci": "npm run lint && npm run cov && node scripts/build_test.js",
|
66
44
|
"contributor": "git-contributor",
|
67
45
|
"clean": "rm -rf dist",
|
@@ -91,7 +69,7 @@
|
|
91
69
|
"@types/qs": "^6.9.7",
|
92
70
|
"@types/selfsigned": "^2.0.1",
|
93
71
|
"@types/tar-stream": "^2.2.2",
|
94
|
-
"@vitest/coverage-v8": "
|
72
|
+
"@vitest/coverage-v8": "beta",
|
95
73
|
"busboy": "^1.6.0",
|
96
74
|
"cross-env": "^7.0.3",
|
97
75
|
"eslint": "^8.25.0",
|
@@ -101,14 +79,39 @@
|
|
101
79
|
"proxy": "^1.0.2",
|
102
80
|
"selfsigned": "^2.0.1",
|
103
81
|
"tar-stream": "^2.2.0",
|
104
|
-
"tshy": "^1.0.0
|
82
|
+
"tshy": "^1.0.0",
|
105
83
|
"tshy-after": "^1.0.0",
|
106
84
|
"typescript": "^5.0.4",
|
107
|
-
"vitest": "
|
85
|
+
"vitest": "beta"
|
108
86
|
},
|
109
87
|
"engines": {
|
110
88
|
"node": ">= 14.19.3"
|
111
89
|
},
|
112
90
|
"license": "MIT",
|
113
|
-
"
|
91
|
+
"type": "module",
|
92
|
+
"tshy": {
|
93
|
+
"exports": {
|
94
|
+
".": "./src/index.ts",
|
95
|
+
"./package.json": "./package.json"
|
96
|
+
}
|
97
|
+
},
|
98
|
+
"exports": {
|
99
|
+
".": {
|
100
|
+
"import": {
|
101
|
+
"types": "./dist/esm/index.d.ts",
|
102
|
+
"default": "./dist/esm/index.js"
|
103
|
+
},
|
104
|
+
"require": {
|
105
|
+
"types": "./dist/commonjs/index.d.ts",
|
106
|
+
"default": "./dist/commonjs/index.js"
|
107
|
+
}
|
108
|
+
},
|
109
|
+
"./package.json": "./package.json"
|
110
|
+
},
|
111
|
+
"files": [
|
112
|
+
"dist",
|
113
|
+
"src"
|
114
|
+
],
|
115
|
+
"types": "./dist/commonjs/index.d.ts",
|
116
|
+
"main": "./dist/commonjs/index.js"
|
114
117
|
}
|
package/src/HttpAgent.ts
CHANGED
@@ -77,7 +77,7 @@ export class HttpAgent extends Agent {
|
|
77
77
|
}
|
78
78
|
const family = isIP(hostname);
|
79
79
|
if (family === 4 || family === 6) {
|
80
|
-
// if request hostname is ip, custom lookup won't
|
80
|
+
// if request hostname is ip, custom lookup won't execute
|
81
81
|
if (!this.#checkAddress(hostname, family)) {
|
82
82
|
throw new IllegalAddressError(hostname, hostname, family);
|
83
83
|
}
|
package/src/HttpClient.ts
CHANGED
@@ -20,6 +20,8 @@ import {
|
|
20
20
|
FormData as FormDataNative,
|
21
21
|
request as undiciRequest,
|
22
22
|
Dispatcher,
|
23
|
+
Agent,
|
24
|
+
getGlobalDispatcher,
|
23
25
|
} from 'undici';
|
24
26
|
import { FormData as FormDataNode } from 'formdata-node';
|
25
27
|
import { FormDataEncoder } from 'form-data-encoder';
|
@@ -36,6 +38,7 @@ import { RawResponseWithMeta, HttpClientResponse, SocketInfo } from './Response.
|
|
36
38
|
import { parseJSON, sleep, digestAuthHeader, globalId, performanceTime, isReadable } from './utils.js';
|
37
39
|
import symbols from './symbols.js';
|
38
40
|
import { initDiagnosticsChannel } from './diagnosticsChannel.js';
|
41
|
+
import { HttpClientConnectTimeoutError, HttpClientRequestTimeoutError } from './HttpClientError.js';
|
39
42
|
|
40
43
|
type Exists<T> = T extends undefined ? never : T;
|
41
44
|
type UndiciRequestOption = Exists<Parameters<typeof undiciRequest>[1]>;
|
@@ -86,11 +89,14 @@ export type ClientOptions = {
|
|
86
89
|
* An 'error' event is emitted if verification fails.Default: true.
|
87
90
|
*/
|
88
91
|
rejectUnauthorized?: boolean;
|
89
|
-
|
90
92
|
/**
|
91
93
|
* socketPath string | null (optional) - Default: null - An IPC endpoint, either Unix domain socket or Windows named pipe
|
92
94
|
*/
|
93
95
|
socketPath?: string | null;
|
96
|
+
/**
|
97
|
+
* connect timeout, default is 10000ms
|
98
|
+
*/
|
99
|
+
timeout?: number;
|
94
100
|
},
|
95
101
|
};
|
96
102
|
|
@@ -116,15 +122,6 @@ class BlobFromStream {
|
|
116
122
|
}
|
117
123
|
}
|
118
124
|
|
119
|
-
class HttpClientRequestTimeoutError extends Error {
|
120
|
-
constructor(timeout: number, options: ErrorOptions) {
|
121
|
-
const message = `Request timeout for ${timeout} ms`;
|
122
|
-
super(message, options);
|
123
|
-
this.name = this.constructor.name;
|
124
|
-
Error.captureStackTrace(this, this.constructor);
|
125
|
-
}
|
126
|
-
}
|
127
|
-
|
128
125
|
export const HEADER_USER_AGENT = createUserAgent('node-urllib', 'VERSION');
|
129
126
|
|
130
127
|
function getFileName(stream: Readable) {
|
@@ -168,16 +165,28 @@ export class HttpClient extends EventEmitter {
|
|
168
165
|
constructor(clientOptions?: ClientOptions) {
|
169
166
|
super();
|
170
167
|
this.#defaultArgs = clientOptions?.defaultArgs;
|
171
|
-
if (clientOptions?.lookup || clientOptions?.checkAddress
|
168
|
+
if (clientOptions?.lookup || clientOptions?.checkAddress) {
|
172
169
|
this.#dispatcher = new HttpAgent({
|
173
170
|
lookup: clientOptions.lookup,
|
174
171
|
checkAddress: clientOptions.checkAddress,
|
175
172
|
connect: clientOptions.connect,
|
176
173
|
});
|
174
|
+
} else if (clientOptions?.connect) {
|
175
|
+
this.#dispatcher = new Agent({
|
176
|
+
connect: clientOptions.connect,
|
177
|
+
});
|
177
178
|
}
|
178
179
|
initDiagnosticsChannel();
|
179
180
|
}
|
180
181
|
|
182
|
+
getDispatcher() {
|
183
|
+
return this.#dispatcher ?? getGlobalDispatcher();
|
184
|
+
}
|
185
|
+
|
186
|
+
setDispatcher(dispatcher: Dispatcher) {
|
187
|
+
this.#dispatcher = dispatcher;
|
188
|
+
}
|
189
|
+
|
181
190
|
async request<T = any>(url: RequestURL, options?: RequestOptions) {
|
182
191
|
return await this.#requestInternal<T>(url, options);
|
183
192
|
}
|
@@ -636,13 +645,15 @@ export class HttpClient extends EventEmitter {
|
|
636
645
|
}
|
637
646
|
|
638
647
|
return clientResponse;
|
639
|
-
} catch (
|
640
|
-
debug('Request#%d throw error: %s', requestId,
|
641
|
-
let err =
|
648
|
+
} catch (rawError: any) {
|
649
|
+
debug('Request#%d throw error: %s', requestId, rawError);
|
650
|
+
let err = rawError;
|
642
651
|
if (err.name === 'HeadersTimeoutError') {
|
643
|
-
err = new HttpClientRequestTimeoutError(headersTimeout, { cause:
|
652
|
+
err = new HttpClientRequestTimeoutError(headersTimeout, { cause: err });
|
644
653
|
} else if (err.name === 'BodyTimeoutError') {
|
645
|
-
err = new HttpClientRequestTimeoutError(bodyTimeout, { cause:
|
654
|
+
err = new HttpClientRequestTimeoutError(bodyTimeout, { cause: err });
|
655
|
+
} else if (err.code === 'UND_ERR_CONNECT_TIMEOUT') {
|
656
|
+
err = new HttpClientConnectTimeoutError(err.message, err.code, { cause: err });
|
646
657
|
} else if (err.code === 'UND_ERR_SOCKET' || err.code === 'ECONNRESET') {
|
647
658
|
// auto retry on socket error, https://github.com/node-modules/urllib/issues/454
|
648
659
|
if (args.socketErrorRetry > 0 && requestContext.socketErrorRetries < args.socketErrorRetry) {
|
@@ -664,7 +675,7 @@ export class HttpClient extends EventEmitter {
|
|
664
675
|
res.requestUrls.push(requestUrl.href);
|
665
676
|
}
|
666
677
|
res.rt = performanceTime(requestStartTime);
|
667
|
-
this.#updateSocketInfo(socketInfo, internalOpaque);
|
678
|
+
this.#updateSocketInfo(socketInfo, internalOpaque, rawError);
|
668
679
|
|
669
680
|
channels.response.publish({
|
670
681
|
request: reqMeta,
|
@@ -687,21 +698,38 @@ export class HttpClient extends EventEmitter {
|
|
687
698
|
}
|
688
699
|
}
|
689
700
|
|
690
|
-
#updateSocketInfo(socketInfo: SocketInfo, internalOpaque: any) {
|
691
|
-
const socket = internalOpaque[symbols.kRequestSocket];
|
701
|
+
#updateSocketInfo(socketInfo: SocketInfo, internalOpaque: any, err?: any) {
|
702
|
+
const socket = internalOpaque[symbols.kRequestSocket] ?? err?.[symbols.kErrorSocket];
|
692
703
|
if (socket) {
|
693
704
|
socketInfo.id = socket[symbols.kSocketId];
|
694
705
|
socketInfo.handledRequests = socket[symbols.kHandledRequests];
|
695
706
|
socketInfo.handledResponses = socket[symbols.kHandledResponses];
|
696
|
-
|
697
|
-
|
698
|
-
|
699
|
-
|
700
|
-
|
707
|
+
if (socket[symbols.kSocketLocalAddress]) {
|
708
|
+
socketInfo.localAddress = socket[symbols.kSocketLocalAddress];
|
709
|
+
socketInfo.localPort = socket[symbols.kSocketLocalPort];
|
710
|
+
}
|
711
|
+
if (socket.remoteAddress) {
|
712
|
+
socketInfo.remoteAddress = socket.remoteAddress;
|
713
|
+
socketInfo.remotePort = socket.remotePort;
|
714
|
+
socketInfo.remoteFamily = socket.remoteFamily;
|
715
|
+
}
|
701
716
|
socketInfo.bytesRead = socket.bytesRead;
|
702
717
|
socketInfo.bytesWritten = socket.bytesWritten;
|
703
|
-
|
704
|
-
|
718
|
+
if (socket[symbols.kSocketConnectErrorTime]) {
|
719
|
+
socketInfo.connectErrorTime = socket[symbols.kSocketConnectErrorTime];
|
720
|
+
if (Array.isArray(socket.autoSelectFamilyAttemptedAddresses)) {
|
721
|
+
socketInfo.attemptedRemoteAddresses = socket.autoSelectFamilyAttemptedAddresses;
|
722
|
+
}
|
723
|
+
socketInfo.connectProtocol = socket[symbols.kSocketConnectProtocol];
|
724
|
+
socketInfo.connectHost = socket[symbols.kSocketConnectHost];
|
725
|
+
socketInfo.connectPort = socket[symbols.kSocketConnectPort];
|
726
|
+
}
|
727
|
+
if (socket[symbols.kSocketConnectedTime]) {
|
728
|
+
socketInfo.connectedTime = socket[symbols.kSocketConnectedTime];
|
729
|
+
}
|
730
|
+
if (socket[symbols.kSocketRequestEndTime]) {
|
731
|
+
socketInfo.lastRequestEndTime = socket[symbols.kSocketRequestEndTime];
|
732
|
+
}
|
705
733
|
socket[symbols.kSocketRequestEndTime] = new Date();
|
706
734
|
}
|
707
735
|
}
|