urllib 3.20.0 → 3.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -11,6 +11,7 @@ import { createReadStream } from 'node:fs';
11
11
  import { format as urlFormat } from 'node:url';
12
12
  import { performance } from 'node:perf_hooks';
13
13
  import { FormData as FormDataNative, request as undiciRequest, Agent, getGlobalDispatcher, } from 'undici';
14
+ import undiciSymbols from 'undici/lib/core/symbols.js';
14
15
  import { FormData as FormDataNode } from 'formdata-node';
15
16
  import { FormDataEncoder } from 'form-data-encoder';
16
17
  import createUserAgent from 'default-user-agent';
@@ -23,6 +24,7 @@ import { HttpAgent } from './HttpAgent.js';
23
24
  import { parseJSON, sleep, digestAuthHeader, globalId, performanceTime, isReadable } from './utils.js';
24
25
  import symbols from './symbols.js';
25
26
  import { initDiagnosticsChannel } from './diagnosticsChannel.js';
27
+ import { HttpClientConnectTimeoutError, HttpClientRequestTimeoutError } from './HttpClientError.js';
26
28
  const PROTO_RE = /^https?:\/\//i;
27
29
  const FormData = FormDataNative ?? FormDataNode;
28
30
  // impl promise pipeline on Node.js 14
@@ -59,15 +61,7 @@ class BlobFromStream {
59
61
  return 'Blob';
60
62
  }
61
63
  }
62
- class HttpClientRequestTimeoutError extends Error {
63
- constructor(timeout, options) {
64
- const message = `Request timeout for ${timeout} ms`;
65
- super(message, options);
66
- this.name = this.constructor.name;
67
- Error.captureStackTrace(this, this.constructor);
68
- }
69
- }
70
- export const HEADER_USER_AGENT = createUserAgent('node-urllib', '3.20.0');
64
+ export const HEADER_USER_AGENT = createUserAgent('node-urllib', '3.22.0');
71
65
  function getFileName(stream) {
72
66
  const filePath = stream.path;
73
67
  if (filePath) {
@@ -108,10 +102,31 @@ export class HttpClient extends EventEmitter {
108
102
  setDispatcher(dispatcher) {
109
103
  this.#dispatcher = dispatcher;
110
104
  }
105
+ getDispatcherPoolStats() {
106
+ const agent = this.getDispatcher();
107
+ // origin => Pool Instance
108
+ const clients = agent[undiciSymbols.kClients];
109
+ const poolStatsMap = {};
110
+ for (const [key, ref] of clients) {
111
+ const pool = ref.deref();
112
+ const stats = pool?.stats;
113
+ if (!stats)
114
+ continue;
115
+ poolStatsMap[key] = {
116
+ connected: stats.connected,
117
+ free: stats.free,
118
+ pending: stats.pending,
119
+ queued: stats.queued,
120
+ running: stats.running,
121
+ size: stats.size,
122
+ };
123
+ }
124
+ return poolStatsMap;
125
+ }
111
126
  async request(url, options) {
112
127
  return await this.#requestInternal(url, options);
113
128
  }
114
- // alias to request, keep compatible with urlib@2 HttpClient.curl
129
+ // alias to request, keep compatible with urllib@2 HttpClient.curl
115
130
  async curl(url, options) {
116
131
  return await this.request(url, options);
117
132
  }
@@ -574,14 +589,17 @@ export class HttpClient extends EventEmitter {
574
589
  }
575
590
  return clientResponse;
576
591
  }
577
- catch (e) {
578
- debug('Request#%d throw error: %s', requestId, e);
579
- let err = e;
592
+ catch (rawError) {
593
+ debug('Request#%d throw error: %s', requestId, rawError);
594
+ let err = rawError;
580
595
  if (err.name === 'HeadersTimeoutError') {
581
- err = new HttpClientRequestTimeoutError(headersTimeout, { cause: e });
596
+ err = new HttpClientRequestTimeoutError(headersTimeout, { cause: err });
582
597
  }
583
598
  else if (err.name === 'BodyTimeoutError') {
584
- err = new HttpClientRequestTimeoutError(bodyTimeout, { cause: e });
599
+ err = new HttpClientRequestTimeoutError(bodyTimeout, { cause: err });
600
+ }
601
+ else if (err.code === 'UND_ERR_CONNECT_TIMEOUT') {
602
+ err = new HttpClientConnectTimeoutError(err.message, err.code, { cause: err });
585
603
  }
586
604
  else if (err.code === 'UND_ERR_SOCKET' || err.code === 'ECONNRESET') {
587
605
  // auto retry on socket error, https://github.com/node-modules/urllib/issues/454
@@ -604,7 +622,7 @@ export class HttpClient extends EventEmitter {
604
622
  res.requestUrls.push(requestUrl.href);
605
623
  }
606
624
  res.rt = performanceTime(requestStartTime);
607
- this.#updateSocketInfo(socketInfo, internalOpaque);
625
+ this.#updateSocketInfo(socketInfo, internalOpaque, rawError);
608
626
  channels.response.publish({
609
627
  request: reqMeta,
610
628
  response: res,
@@ -625,21 +643,38 @@ export class HttpClient extends EventEmitter {
625
643
  throw err;
626
644
  }
627
645
  }
628
- #updateSocketInfo(socketInfo, internalOpaque) {
629
- const socket = internalOpaque[symbols.kRequestSocket];
646
+ #updateSocketInfo(socketInfo, internalOpaque, err) {
647
+ const socket = internalOpaque[symbols.kRequestSocket] ?? err?.[symbols.kErrorSocket];
630
648
  if (socket) {
631
649
  socketInfo.id = socket[symbols.kSocketId];
632
650
  socketInfo.handledRequests = socket[symbols.kHandledRequests];
633
651
  socketInfo.handledResponses = socket[symbols.kHandledResponses];
634
- socketInfo.localAddress = socket[symbols.kSocketLocalAddress];
635
- socketInfo.localPort = socket[symbols.kSocketLocalPort];
636
- socketInfo.remoteAddress = socket.remoteAddress;
637
- socketInfo.remotePort = socket.remotePort;
638
- socketInfo.remoteFamily = socket.remoteFamily;
652
+ if (socket[symbols.kSocketLocalAddress]) {
653
+ socketInfo.localAddress = socket[symbols.kSocketLocalAddress];
654
+ socketInfo.localPort = socket[symbols.kSocketLocalPort];
655
+ }
656
+ if (socket.remoteAddress) {
657
+ socketInfo.remoteAddress = socket.remoteAddress;
658
+ socketInfo.remotePort = socket.remotePort;
659
+ socketInfo.remoteFamily = socket.remoteFamily;
660
+ }
639
661
  socketInfo.bytesRead = socket.bytesRead;
640
662
  socketInfo.bytesWritten = socket.bytesWritten;
641
- socketInfo.connectedTime = socket[symbols.kSocketConnectedTime];
642
- socketInfo.lastRequestEndTime = socket[symbols.kSocketRequestEndTime];
663
+ if (socket[symbols.kSocketConnectErrorTime]) {
664
+ socketInfo.connectErrorTime = socket[symbols.kSocketConnectErrorTime];
665
+ if (Array.isArray(socket.autoSelectFamilyAttemptedAddresses)) {
666
+ socketInfo.attemptedRemoteAddresses = socket.autoSelectFamilyAttemptedAddresses;
667
+ }
668
+ socketInfo.connectProtocol = socket[symbols.kSocketConnectProtocol];
669
+ socketInfo.connectHost = socket[symbols.kSocketConnectHost];
670
+ socketInfo.connectPort = socket[symbols.kSocketConnectPort];
671
+ }
672
+ if (socket[symbols.kSocketConnectedTime]) {
673
+ socketInfo.connectedTime = socket[symbols.kSocketConnectedTime];
674
+ }
675
+ if (socket[symbols.kSocketRequestEndTime]) {
676
+ socketInfo.lastRequestEndTime = socket[symbols.kSocketRequestEndTime];
677
+ }
643
678
  socket[symbols.kSocketRequestEndTime] = new Date();
644
679
  }
645
680
  }
@@ -0,0 +1,19 @@
1
+ import type { RawResponseWithMeta, SocketInfo } from './Response.js';
2
+ import type { IncomingHttpHeaders } from './IncomingHttpHeaders.js';
3
+ interface ErrorOptions {
4
+ cause?: Error;
5
+ }
6
+ export declare class HttpClientRequestError extends Error {
7
+ status?: number;
8
+ headers?: IncomingHttpHeaders;
9
+ socket?: SocketInfo;
10
+ res?: RawResponseWithMeta;
11
+ }
12
+ export declare class HttpClientRequestTimeoutError extends HttpClientRequestError {
13
+ constructor(timeout: number, options: ErrorOptions);
14
+ }
15
+ export declare class HttpClientConnectTimeoutError extends HttpClientRequestError {
16
+ code: string;
17
+ constructor(message: string, code: string, options: ErrorOptions);
18
+ }
19
+ export {};
@@ -0,0 +1,23 @@
1
+ export class HttpClientRequestError extends Error {
2
+ status;
3
+ headers;
4
+ socket;
5
+ res;
6
+ }
7
+ export class HttpClientRequestTimeoutError extends HttpClientRequestError {
8
+ constructor(timeout, options) {
9
+ const message = `Request timeout for ${timeout} ms`;
10
+ super(message, options);
11
+ this.name = this.constructor.name;
12
+ Error.captureStackTrace(this, this.constructor);
13
+ }
14
+ }
15
+ export class HttpClientConnectTimeoutError extends HttpClientRequestError {
16
+ code;
17
+ constructor(message, code, options) {
18
+ super(message, options);
19
+ this.name = this.constructor.name;
20
+ this.code = code;
21
+ Error.captureStackTrace(this, this.constructor);
22
+ }
23
+ }
@@ -13,7 +13,12 @@ export type SocketInfo = {
13
13
  handledRequests: number;
14
14
  handledResponses: number;
15
15
  connectedTime?: Date;
16
+ connectErrorTime?: Date;
16
17
  lastRequestEndTime?: Date;
18
+ attemptedRemoteAddresses?: string[];
19
+ connectProtocol?: string;
20
+ connectHost?: string;
21
+ connectPort?: string;
17
22
  };
18
23
  /**
19
24
  * https://eggjs.org/en/core/httpclient.html#timing-boolean
@@ -1,6 +1,7 @@
1
1
  import diagnosticsChannel from 'node:diagnostics_channel';
2
2
  import { performance } from 'node:perf_hooks';
3
3
  import { debuglog } from 'node:util';
4
+ import { Socket } from 'node:net';
4
5
  import symbols from './symbols.js';
5
6
  import { globalId, performanceTime } from './utils.js';
6
7
  const debug = debuglog('urllib:DiagnosticsChannel');
@@ -32,8 +33,23 @@ function formatSocket(socket) {
32
33
  localPort: socket[symbols.kSocketLocalPort],
33
34
  remoteAddress: socket.remoteAddress,
34
35
  remotePort: socket.remotePort,
36
+ attemptedAddresses: socket.autoSelectFamilyAttemptedAddresses,
37
+ connecting: socket.connecting,
35
38
  };
36
39
  }
40
+ // make sure error contains socket info
41
+ const kDestroy = Symbol('kDestroy');
42
+ Socket.prototype[kDestroy] = Socket.prototype.destroy;
43
+ Socket.prototype.destroy = function (err) {
44
+ if (err) {
45
+ Object.defineProperty(err, symbols.kErrorSocket, {
46
+ // don't show on console log
47
+ enumerable: false,
48
+ value: this,
49
+ });
50
+ }
51
+ return this[kDestroy](err);
52
+ };
37
53
  export function initDiagnosticsChannel() {
38
54
  // makre sure init global DiagnosticsChannel once
39
55
  if (initedDiagnosticsChannel)
@@ -63,10 +79,34 @@ export function initDiagnosticsChannel() {
63
79
  opaque[symbols.kRequestTiming].queuing = performanceTime(opaque[symbols.kRequestStartTime]);
64
80
  });
65
81
  // diagnosticsChannel.channel('undici:client:beforeConnect')
66
- // diagnosticsChannel.channel('undici:client:connectError')
82
+ subscribe('undici:client:connectError', (message, name) => {
83
+ const { error, connectParams } = message;
84
+ let { socket } = message;
85
+ if (!socket && error[symbols.kErrorSocket]) {
86
+ socket = error[symbols.kErrorSocket];
87
+ }
88
+ if (socket) {
89
+ socket[symbols.kSocketId] = globalId('UndiciSocket');
90
+ socket[symbols.kSocketConnectErrorTime] = new Date();
91
+ socket[symbols.kHandledRequests] = 0;
92
+ socket[symbols.kHandledResponses] = 0;
93
+ // copy local address to symbol, avoid them be reset after request error throw
94
+ if (socket.localAddress) {
95
+ socket[symbols.kSocketLocalAddress] = socket.localAddress;
96
+ socket[symbols.kSocketLocalPort] = socket.localPort;
97
+ }
98
+ socket[symbols.kSocketConnectProtocol] = connectParams.protocol;
99
+ socket[symbols.kSocketConnectHost] = connectParams.host;
100
+ socket[symbols.kSocketConnectPort] = connectParams.port;
101
+ debug('[%s] Socket#%d connectError, connectParams: %o, error: %s, (sock: %o)', name, socket[symbols.kSocketId], connectParams, error.message, formatSocket(socket));
102
+ }
103
+ else {
104
+ debug('[%s] connectError, connectParams: %o, error: %o', name, connectParams, error);
105
+ }
106
+ });
67
107
  // This message is published after a connection is established.
68
108
  subscribe('undici:client:connected', (message, name) => {
69
- const { socket } = message;
109
+ const { socket, connectParams } = message;
70
110
  socket[symbols.kSocketId] = globalId('UndiciSocket');
71
111
  socket[symbols.kSocketStartTime] = performance.now();
72
112
  socket[symbols.kSocketConnectedTime] = new Date();
@@ -75,6 +115,9 @@ export function initDiagnosticsChannel() {
75
115
  // copy local address to symbol, avoid them be reset after request error throw
76
116
  socket[symbols.kSocketLocalAddress] = socket.localAddress;
77
117
  socket[symbols.kSocketLocalPort] = socket.localPort;
118
+ socket[symbols.kSocketConnectProtocol] = connectParams.protocol;
119
+ socket[symbols.kSocketConnectHost] = connectParams.host;
120
+ socket[symbols.kSocketConnectPort] = connectParams.port;
78
121
  debug('[%s] Socket#%d connected (sock: %o)', name, socket[symbols.kSocketId], formatSocket(socket));
79
122
  });
80
123
  // This message is published right before the first byte of the request is written to the socket.
@@ -1,4 +1,6 @@
1
+ import { HttpClient } from './HttpClient.js';
1
2
  import { RequestOptions, RequestURL } from './Request.js';
3
+ export declare function getDefaultHttpClient(): HttpClient;
2
4
  export declare function request<T = any>(url: RequestURL, options?: RequestOptions): Promise<import("./Response.js").HttpClientResponse<T>>;
3
5
  export declare function curl<T = any>(url: RequestURL, options?: RequestOptions): Promise<import("./Response.js").HttpClientResponse<T>>;
4
6
  export { MockAgent, ProxyAgent, Agent, Dispatcher, setGlobalDispatcher, getGlobalDispatcher, } from 'undici';
@@ -6,6 +8,7 @@ export { HttpClient, HttpClient as HttpClient2, HEADER_USER_AGENT as USER_AGENT,
6
8
  export { RequestOptions, RequestOptions as RequestOptions2, RequestURL, HttpMethod, FixJSONCtlCharsHandler, FixJSONCtlChars, } from './Request.js';
7
9
  export { SocketInfo, Timing, RawResponseWithMeta, HttpClientResponse, } from './Response.js';
8
10
  export { IncomingHttpHeaders, } from './IncomingHttpHeaders.js';
11
+ export * from './HttpClientError.js';
9
12
  declare const _default: {
10
13
  request: typeof request;
11
14
  curl: typeof curl;
package/dist/esm/index.js CHANGED
@@ -1,22 +1,25 @@
1
1
  import LRU from 'ylru';
2
2
  import { HttpClient, HEADER_USER_AGENT } from './HttpClient.js';
3
- let httpclient;
4
- const domainSocketHttpclients = new LRU(50);
3
+ let httpClient;
4
+ const domainSocketHttpClients = new LRU(50);
5
+ export function getDefaultHttpClient() {
6
+ if (!httpClient) {
7
+ httpClient = new HttpClient();
8
+ }
9
+ return httpClient;
10
+ }
5
11
  export async function request(url, options) {
6
12
  if (options?.socketPath) {
7
- let domainSocketHttpclient = domainSocketHttpclients.get(options.socketPath);
13
+ let domainSocketHttpclient = domainSocketHttpClients.get(options.socketPath);
8
14
  if (!domainSocketHttpclient) {
9
15
  domainSocketHttpclient = new HttpClient({
10
16
  connect: { socketPath: options.socketPath },
11
17
  });
12
- domainSocketHttpclients.set(options.socketPath, domainSocketHttpclient);
18
+ domainSocketHttpClients.set(options.socketPath, domainSocketHttpclient);
13
19
  }
14
20
  return await domainSocketHttpclient.request(url, options);
15
21
  }
16
- if (!httpclient) {
17
- httpclient = new HttpClient({});
18
- }
19
- return await httpclient.request(url, options);
22
+ return await getDefaultHttpClient().request(url, options);
20
23
  }
21
24
  // export curl method is keep compatible with urllib.curl()
22
25
  // ```ts
@@ -27,8 +30,9 @@ export async function curl(url, options) {
27
30
  return await request(url, options);
28
31
  }
29
32
  export { MockAgent, ProxyAgent, Agent, Dispatcher, setGlobalDispatcher, getGlobalDispatcher, } from 'undici';
30
- // HttpClient2 is keep compatible with urlib@2 HttpClient2
33
+ // HttpClient2 is keep compatible with urllib@2 HttpClient2
31
34
  export { HttpClient, HttpClient as HttpClient2, HEADER_USER_AGENT as USER_AGENT, } from './HttpClient.js';
35
+ export * from './HttpClientError.js';
32
36
  export default {
33
37
  request,
34
38
  curl,
@@ -2,9 +2,13 @@ declare const _default: {
2
2
  kSocketId: symbol;
3
3
  kSocketStartTime: symbol;
4
4
  kSocketConnectedTime: symbol;
5
+ kSocketConnectErrorTime: symbol;
5
6
  kSocketRequestEndTime: symbol;
6
7
  kSocketLocalAddress: symbol;
7
8
  kSocketLocalPort: symbol;
9
+ kSocketConnectHost: symbol;
10
+ kSocketConnectPort: symbol;
11
+ kSocketConnectProtocol: symbol;
8
12
  kHandledRequests: symbol;
9
13
  kHandledResponses: symbol;
10
14
  kRequestSocket: symbol;
@@ -13,5 +17,6 @@ declare const _default: {
13
17
  kEnableRequestTiming: symbol;
14
18
  kRequestTiming: symbol;
15
19
  kRequestOriginalOpaque: symbol;
20
+ kErrorSocket: symbol;
16
21
  };
17
22
  export default _default;
@@ -2,9 +2,13 @@ export default {
2
2
  kSocketId: Symbol('socket id'),
3
3
  kSocketStartTime: Symbol('socket start time'),
4
4
  kSocketConnectedTime: Symbol('socket connected time'),
5
+ kSocketConnectErrorTime: Symbol('socket connectError time'),
5
6
  kSocketRequestEndTime: Symbol('socket request end time'),
6
7
  kSocketLocalAddress: Symbol('socket local address'),
7
8
  kSocketLocalPort: Symbol('socket local port'),
9
+ kSocketConnectHost: Symbol('socket connect params: host'),
10
+ kSocketConnectPort: Symbol('socket connect params: port'),
11
+ kSocketConnectProtocol: Symbol('socket connect params: protocol'),
8
12
  kHandledRequests: Symbol('handled requests per socket'),
9
13
  kHandledResponses: Symbol('handled responses per socket'),
10
14
  kRequestSocket: Symbol('request on the socket'),
@@ -13,4 +17,5 @@ export default {
13
17
  kEnableRequestTiming: Symbol('enable request timing or not'),
14
18
  kRequestTiming: Symbol('request timing'),
15
19
  kRequestOriginalOpaque: Symbol('request original opaque'),
20
+ kErrorSocket: Symbol('socket of error'),
16
21
  };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "urllib",
3
- "version": "3.20.0",
3
+ "version": "3.22.0",
4
4
  "publishConfig": {
5
5
  "tag": "latest"
6
6
  },
@@ -32,8 +32,9 @@
32
32
  "build:esm:test": "cd test/esm && rm -rf node_modules && npm link ../.. && node index.js",
33
33
  "build:mts:test": "cd test/mts && rm -rf node_modules && npm link ../.. && tsc",
34
34
  "build:test": "npm run build && npm run build:cjs:test && npm run build:esm:test && npm run build:mts:test && npm run test-tsc",
35
- "test-tsc": "npm run test-tsc:cjs && npm run test-tsc:esm",
35
+ "test-tsc": "npm run test-tsc:cjs:es2021 && npm run test-tsc:cjs && npm run test-tsc:esm",
36
36
  "test-tsc:cjs": "cd test/fixtures/ts && rm -rf node_modules && npm link ../../.. && npm run build",
37
+ "test-tsc:cjs:es2021": "cd test/fixtures/ts-cjs-es2021 && rm -rf node_modules && npm link ../../.. && npm run build",
37
38
  "test-tsc:esm": "cd test/fixtures/ts-esm && rm -rf node_modules && npm link ../../.. && npm run build",
38
39
  "test": "npm run lint && vitest run",
39
40
  "test-keepalive": "cross-env TEST_KEEPALIVE_COUNT=50 vitest run --test-timeout 180000 keep-alive-header.test.ts",
@@ -68,7 +69,7 @@
68
69
  "@types/qs": "^6.9.7",
69
70
  "@types/selfsigned": "^2.0.1",
70
71
  "@types/tar-stream": "^2.2.2",
71
- "@vitest/coverage-v8": "beta",
72
+ "@vitest/coverage-v8": "^1.0.1",
72
73
  "busboy": "^1.6.0",
73
74
  "cross-env": "^7.0.3",
74
75
  "eslint": "^8.25.0",
@@ -81,7 +82,7 @@
81
82
  "tshy": "^1.0.0",
82
83
  "tshy-after": "^1.0.0",
83
84
  "typescript": "^5.0.4",
84
- "vitest": "beta"
85
+ "vitest": "^1.0.1"
85
86
  },
86
87
  "engines": {
87
88
  "node": ">= 14.19.3"
package/src/HttpAgent.ts CHANGED
@@ -77,7 +77,7 @@ export class HttpAgent extends Agent {
77
77
  }
78
78
  const family = isIP(hostname);
79
79
  if (family === 4 || family === 6) {
80
- // if request hostname is ip, custom lookup won't excute
80
+ // if request hostname is ip, custom lookup won't execute
81
81
  if (!this.#checkAddress(hostname, family)) {
82
82
  throw new IllegalAddressError(hostname, hostname, family);
83
83
  }
package/src/HttpClient.ts CHANGED
@@ -22,7 +22,9 @@ import {
22
22
  Dispatcher,
23
23
  Agent,
24
24
  getGlobalDispatcher,
25
+ Pool,
25
26
  } from 'undici';
27
+ import undiciSymbols from 'undici/lib/core/symbols.js';
26
28
  import { FormData as FormDataNode } from 'formdata-node';
27
29
  import { FormDataEncoder } from 'form-data-encoder';
28
30
  import createUserAgent from 'default-user-agent';
@@ -38,6 +40,7 @@ import { RawResponseWithMeta, HttpClientResponse, SocketInfo } from './Response.
38
40
  import { parseJSON, sleep, digestAuthHeader, globalId, performanceTime, isReadable } from './utils.js';
39
41
  import symbols from './symbols.js';
40
42
  import { initDiagnosticsChannel } from './diagnosticsChannel.js';
43
+ import { HttpClientConnectTimeoutError, HttpClientRequestTimeoutError } from './HttpClientError.js';
41
44
 
42
45
  type Exists<T> = T extends undefined ? never : T;
43
46
  type UndiciRequestOption = Exists<Parameters<typeof undiciRequest>[1]>;
@@ -121,15 +124,6 @@ class BlobFromStream {
121
124
  }
122
125
  }
123
126
 
124
- class HttpClientRequestTimeoutError extends Error {
125
- constructor(timeout: number, options: ErrorOptions) {
126
- const message = `Request timeout for ${timeout} ms`;
127
- super(message, options);
128
- this.name = this.constructor.name;
129
- Error.captureStackTrace(this, this.constructor);
130
- }
131
- }
132
-
133
127
  export const HEADER_USER_AGENT = createUserAgent('node-urllib', 'VERSION');
134
128
 
135
129
  function getFileName(stream: Readable) {
@@ -144,7 +138,7 @@ function defaultIsRetry(response: HttpClientResponse) {
144
138
  return response.status >= 500;
145
139
  }
146
140
 
147
- type RequestContext = {
141
+ export type RequestContext = {
148
142
  retries: number;
149
143
  socketErrorRetries: number;
150
144
  requestStartTime?: number;
@@ -165,6 +159,20 @@ export type ResponseDiagnosticsMessage = {
165
159
  error?: Error;
166
160
  };
167
161
 
162
+ export interface PoolStat {
163
+ /** Number of open socket connections in this pool. */
164
+ connected: number;
165
+ /** Number of open socket connections in this pool that do not have an active request. */
166
+ free: number;
167
+ /** Number of pending requests across all clients in this pool. */
168
+ pending: number;
169
+ /** Number of queued requests across all clients in this pool. */
170
+ queued: number;
171
+ /** Number of currently active requests across all clients in this pool. */
172
+ running: number;
173
+ /** Number of active, pending, or queued requests across all clients in this pool. */
174
+ size: number;
175
+ }
168
176
 
169
177
  export class HttpClient extends EventEmitter {
170
178
  #defaultArgs?: RequestOptions;
@@ -195,11 +203,32 @@ export class HttpClient extends EventEmitter {
195
203
  this.#dispatcher = dispatcher;
196
204
  }
197
205
 
206
+ getDispatcherPoolStats() {
207
+ const agent = this.getDispatcher();
208
+ // origin => Pool Instance
209
+ const clients: Map<string, WeakRef<Pool>> = agent[undiciSymbols.kClients];
210
+ const poolStatsMap: Record<string, PoolStat> = {};
211
+ for (const [ key, ref ] of clients) {
212
+ const pool = ref.deref();
213
+ const stats = pool?.stats;
214
+ if (!stats) continue;
215
+ poolStatsMap[key] = {
216
+ connected: stats.connected,
217
+ free: stats.free,
218
+ pending: stats.pending,
219
+ queued: stats.queued,
220
+ running: stats.running,
221
+ size: stats.size,
222
+ } satisfies PoolStat;
223
+ }
224
+ return poolStatsMap;
225
+ }
226
+
198
227
  async request<T = any>(url: RequestURL, options?: RequestOptions) {
199
228
  return await this.#requestInternal<T>(url, options);
200
229
  }
201
230
 
202
- // alias to request, keep compatible with urlib@2 HttpClient.curl
231
+ // alias to request, keep compatible with urllib@2 HttpClient.curl
203
232
  async curl<T = any>(url: RequestURL, options?: RequestOptions) {
204
233
  return await this.request<T>(url, options);
205
234
  }
@@ -653,13 +682,15 @@ export class HttpClient extends EventEmitter {
653
682
  }
654
683
 
655
684
  return clientResponse;
656
- } catch (e: any) {
657
- debug('Request#%d throw error: %s', requestId, e);
658
- let err = e;
685
+ } catch (rawError: any) {
686
+ debug('Request#%d throw error: %s', requestId, rawError);
687
+ let err = rawError;
659
688
  if (err.name === 'HeadersTimeoutError') {
660
- err = new HttpClientRequestTimeoutError(headersTimeout, { cause: e });
689
+ err = new HttpClientRequestTimeoutError(headersTimeout, { cause: err });
661
690
  } else if (err.name === 'BodyTimeoutError') {
662
- err = new HttpClientRequestTimeoutError(bodyTimeout, { cause: e });
691
+ err = new HttpClientRequestTimeoutError(bodyTimeout, { cause: err });
692
+ } else if (err.code === 'UND_ERR_CONNECT_TIMEOUT') {
693
+ err = new HttpClientConnectTimeoutError(err.message, err.code, { cause: err });
663
694
  } else if (err.code === 'UND_ERR_SOCKET' || err.code === 'ECONNRESET') {
664
695
  // auto retry on socket error, https://github.com/node-modules/urllib/issues/454
665
696
  if (args.socketErrorRetry > 0 && requestContext.socketErrorRetries < args.socketErrorRetry) {
@@ -681,7 +712,7 @@ export class HttpClient extends EventEmitter {
681
712
  res.requestUrls.push(requestUrl.href);
682
713
  }
683
714
  res.rt = performanceTime(requestStartTime);
684
- this.#updateSocketInfo(socketInfo, internalOpaque);
715
+ this.#updateSocketInfo(socketInfo, internalOpaque, rawError);
685
716
 
686
717
  channels.response.publish({
687
718
  request: reqMeta,
@@ -704,21 +735,38 @@ export class HttpClient extends EventEmitter {
704
735
  }
705
736
  }
706
737
 
707
- #updateSocketInfo(socketInfo: SocketInfo, internalOpaque: any) {
708
- const socket = internalOpaque[symbols.kRequestSocket];
738
+ #updateSocketInfo(socketInfo: SocketInfo, internalOpaque: any, err?: any) {
739
+ const socket = internalOpaque[symbols.kRequestSocket] ?? err?.[symbols.kErrorSocket];
709
740
  if (socket) {
710
741
  socketInfo.id = socket[symbols.kSocketId];
711
742
  socketInfo.handledRequests = socket[symbols.kHandledRequests];
712
743
  socketInfo.handledResponses = socket[symbols.kHandledResponses];
713
- socketInfo.localAddress = socket[symbols.kSocketLocalAddress];
714
- socketInfo.localPort = socket[symbols.kSocketLocalPort];
715
- socketInfo.remoteAddress = socket.remoteAddress;
716
- socketInfo.remotePort = socket.remotePort;
717
- socketInfo.remoteFamily = socket.remoteFamily;
744
+ if (socket[symbols.kSocketLocalAddress]) {
745
+ socketInfo.localAddress = socket[symbols.kSocketLocalAddress];
746
+ socketInfo.localPort = socket[symbols.kSocketLocalPort];
747
+ }
748
+ if (socket.remoteAddress) {
749
+ socketInfo.remoteAddress = socket.remoteAddress;
750
+ socketInfo.remotePort = socket.remotePort;
751
+ socketInfo.remoteFamily = socket.remoteFamily;
752
+ }
718
753
  socketInfo.bytesRead = socket.bytesRead;
719
754
  socketInfo.bytesWritten = socket.bytesWritten;
720
- socketInfo.connectedTime = socket[symbols.kSocketConnectedTime];
721
- socketInfo.lastRequestEndTime = socket[symbols.kSocketRequestEndTime];
755
+ if (socket[symbols.kSocketConnectErrorTime]) {
756
+ socketInfo.connectErrorTime = socket[symbols.kSocketConnectErrorTime];
757
+ if (Array.isArray(socket.autoSelectFamilyAttemptedAddresses)) {
758
+ socketInfo.attemptedRemoteAddresses = socket.autoSelectFamilyAttemptedAddresses;
759
+ }
760
+ socketInfo.connectProtocol = socket[symbols.kSocketConnectProtocol];
761
+ socketInfo.connectHost = socket[symbols.kSocketConnectHost];
762
+ socketInfo.connectPort = socket[symbols.kSocketConnectPort];
763
+ }
764
+ if (socket[symbols.kSocketConnectedTime]) {
765
+ socketInfo.connectedTime = socket[symbols.kSocketConnectedTime];
766
+ }
767
+ if (socket[symbols.kSocketRequestEndTime]) {
768
+ socketInfo.lastRequestEndTime = socket[symbols.kSocketRequestEndTime];
769
+ }
722
770
  socket[symbols.kSocketRequestEndTime] = new Date();
723
771
  }
724
772
  }
@@ -0,0 +1,34 @@
1
+ import type { RawResponseWithMeta, SocketInfo } from './Response.js';
2
+ import type { IncomingHttpHeaders } from './IncomingHttpHeaders.js';
3
+
4
+ // need to support ES2021
5
+ interface ErrorOptions {
6
+ cause?: Error;
7
+ }
8
+
9
+ export class HttpClientRequestError extends Error {
10
+ status?: number;
11
+ headers?: IncomingHttpHeaders;
12
+ socket?: SocketInfo;
13
+ res?: RawResponseWithMeta;
14
+ }
15
+
16
+ export class HttpClientRequestTimeoutError extends HttpClientRequestError {
17
+ constructor(timeout: number, options: ErrorOptions) {
18
+ const message = `Request timeout for ${timeout} ms`;
19
+ super(message, options);
20
+ this.name = this.constructor.name;
21
+ Error.captureStackTrace(this, this.constructor);
22
+ }
23
+ }
24
+
25
+ export class HttpClientConnectTimeoutError extends HttpClientRequestError {
26
+ code: string;
27
+
28
+ constructor(message: string, code: string, options: ErrorOptions) {
29
+ super(message, options);
30
+ this.name = this.constructor.name;
31
+ this.code = code;
32
+ Error.captureStackTrace(this, this.constructor);
33
+ }
34
+ }
package/src/Response.ts CHANGED
@@ -13,7 +13,12 @@ export type SocketInfo = {
13
13
  handledRequests: number;
14
14
  handledResponses: number;
15
15
  connectedTime?: Date;
16
+ connectErrorTime?: Date;
16
17
  lastRequestEndTime?: Date;
18
+ attemptedRemoteAddresses?: string[];
19
+ connectProtocol?: string;
20
+ connectHost?: string;
21
+ connectPort?: string;
17
22
  };
18
23
 
19
24
  /**